From 454fc11a04381120d90fb898aff14d296f9cc888 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Tue, 17 Dec 2024 19:08:45 +0000 Subject: [PATCH 01/54] cli tune for explicit no database fix WIP WIP WIP --- ydb/core/cms/console/configs_dispatcher.cpp | 25 ++++++++++++++-- ydb/core/cms/console/console.h | 29 +++++++++++++++++++ .../cms/console/console__get_yaml_config.cpp | 10 +++++++ .../console/console__replace_yaml_config.cpp | 6 +++- .../cms/console/console_configs_manager.h | 1 + .../cms/console/console_configs_provider.cpp | 12 ++++++++ .../cms/console/console_configs_provider.h | 14 ++++++++- .../console/console_configs_subscriber.cpp | 7 ++++- ydb/core/grpc_services/rpc_dynamic_config.cpp | 3 ++ ydb/core/protos/console_config.proto | 12 ++++++++ ydb/core/protos/feature_flags.proto | 1 + .../yaml_config/public/yaml_config.cpp | 4 +++ ydb/library/yaml_config/public/yaml_config.h | 5 ++++ .../ydb_cli/commands/ydb_dynamic_config.cpp | 3 +- .../lib/ydb_cli/commands/ydb_root_common.cpp | 11 ++++++- .../lib/ydb_cli/commands/ydb_root_common.h | 1 + 16 files changed, 137 insertions(+), 7 deletions(-) diff --git a/ydb/core/cms/console/configs_dispatcher.cpp b/ydb/core/cms/console/configs_dispatcher.cpp index 66847e9b9dcf..5c622b114f88 100644 --- a/ydb/core/cms/console/configs_dispatcher.cpp +++ b/ydb/core/cms/console/configs_dispatcher.cpp @@ -870,9 +870,30 @@ void TConfigsDispatcher::Handle(TEvConsole::TEvConfigSubscriptionNotification::T CurrentConfig = rec.GetConfig(); - const auto& newYamlConfig = rec.GetYamlConfig(); + auto newYamlConfig = rec.GetYamlConfig(); + + bool isYamlChanged = newYamlConfig != YamlConfig || rec.HasDatabaseConfig(); + + if (rec.HasDatabaseConfig()) { + auto in = rec.GetDatabaseConfig(); + auto pos = in.find("config:"); + in = in.substr(pos); + TString newname; + newname += "- description: dbconfig\n"; + newname += " selector: \n"; + newname += " tenant: "; + newname += Labels.at("tenant"); + newname += "\n "; + for (ui64 i = 0; i < in.size(); i++) { + newname += in[i]; + if(in[i] == '\n' && i != 0) + newname += " "; + } - bool isYamlChanged = newYamlConfig != YamlConfig; + newYamlConfig.append(newname); + Cerr << " xxx 4 " << rec.GetDatabaseConfig(); + Cerr << " xxx 5 " << newYamlConfig; + } if (rec.VolatileConfigsSize() != VolatileYamlConfigs.size()) { isYamlChanged = true; diff --git a/ydb/core/cms/console/console.h b/ydb/core/cms/console/console.h index 1cfc641a372e..971efc153393 100644 --- a/ydb/core/cms/console/console.h +++ b/ydb/core/cms/console/console.h @@ -375,6 +375,35 @@ namespace TEvConsole { } } } + + TEvConfigSubscriptionNotification( + ui64 generation, + const NKikimrConfig::TAppConfig &config, + const THashSet &affectedKinds, + const TString &yamlConfig, + const TMap &volatileYamlConfigs, + const NKikimrConfig::TAppConfig &rawConfig, + const TMaybe dbc) + { + Record.SetGeneration(generation); + Record.MutableConfig()->CopyFrom(config); + Record.MutableRawConsoleConfig()->CopyFrom(rawConfig); + for (ui32 kind : affectedKinds) + Record.AddAffectedKinds(kind); + + if (!yamlConfig.empty()) { + Record.SetYamlConfig(yamlConfig); + for (auto &[id, config] : volatileYamlConfigs) { + auto *volatileConfig = Record.AddVolatileConfigs(); + volatileConfig->SetId(id); + volatileConfig->SetConfig(config); + } + } + if (dbc) { + Record.SetDatabaseConfig(*dbc); + } + } + }; /** diff --git a/ydb/core/cms/console/console__get_yaml_config.cpp b/ydb/core/cms/console/console__get_yaml_config.cpp index d25d264c4878..6f491ddff82d 100644 --- a/ydb/core/cms/console/console__get_yaml_config.cpp +++ b/ydb/core/cms/console/console__get_yaml_config.cpp @@ -12,6 +12,7 @@ class TConfigsManager::TTxGetYamlConfig : public TTransactionBaseGet()->Record.HasDatabase() ? TMaybe{Request->Get()->Record.GetDatabase()} : TMaybe{}) { } @@ -19,6 +20,14 @@ class TConfigsManager::TTxGetYamlConfig : public TTransactionBase(); + if (Database) { + if (Self->YamlConfigPerDatabase.contains(*Database)) { + Response->Record.MutableResponse()->set_config(Self->YamlConfigPerDatabase[*Database]); + } + + return true; + } + Response->Record.MutableResponse()->mutable_identity()->set_cluster(Self->ClusterName); Response->Record.MutableResponse()->mutable_identity()->set_version(Self->YamlVersion); Response->Record.MutableResponse()->set_config(Self->YamlConfig); @@ -44,6 +53,7 @@ class TConfigsManager::TTxGetYamlConfig : public TTransactionBase Response; + TMaybe Database; }; ITransaction *TConfigsManager::CreateTxGetYamlConfig(TEvConsole::TEvGetAllConfigsRequest::TPtr &ev) diff --git a/ydb/core/cms/console/console__replace_yaml_config.cpp b/ydb/core/cms/console/console__replace_yaml_config.cpp index 6eeb03ea5e46..ccd3d935fdcb 100644 --- a/ydb/core/cms/console/console__replace_yaml_config.cpp +++ b/ydb/core/cms/console/console__replace_yaml_config.cpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace NKikimr::NConsole { @@ -25,6 +26,7 @@ class TConfigsManager::TTxReplaceYamlConfig : public TTransactionBaseGet()->Record.GetRequest().allow_unknown_fields()) , DryRun(ev->Get()->Record.GetRequest().dry_run()) + , Database(ev->Get()->Record.HasDatabase() ? TMaybe{ev->Get()->Record.GetDatabase()} : TMaybe{}) { } @@ -151,7 +153,7 @@ class TConfigsManager::TTxReplaceYamlConfig : public TTransactionBaseVolatileYamlConfigs.clear(); - auto resp = MakeHolder(Self->YamlConfig); + auto resp = MakeHolder(Self->YamlConfig, Self->YamlConfigPerDatabase); ctx.Send(Self->ConfigsProvider, resp.Release()); } else if (Error && !DryRun) { AuditLogReplaceConfigTransaction( @@ -194,6 +196,8 @@ class TConfigsManager::TTxReplaceYamlConfig : public TTransactionBase Database; + bool WarnDatabaseByPass = false; }; ITransaction *TConfigsManager::CreateTxReplaceYamlConfig(TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev) diff --git a/ydb/core/cms/console/console_configs_manager.h b/ydb/core/cms/console/console_configs_manager.h index a95513965ec5..fbff482a9a8d 100644 --- a/ydb/core/cms/console/console_configs_manager.h +++ b/ydb/core/cms/console/console_configs_manager.h @@ -301,6 +301,7 @@ class TConfigsManager : public TActorBootstrapped { TString ClusterName; ui32 YamlVersion = 0; TString YamlConfig; + THashMap YamlConfigPerDatabase; bool YamlDropped = false; bool YamlReadOnly = true; TMap VolatileYamlConfigs; diff --git a/ydb/core/cms/console/console_configs_provider.cpp b/ydb/core/cms/console/console_configs_provider.cpp index 1f5691941c63..787c9db335e7 100644 --- a/ydb/core/cms/console/console_configs_provider.cpp +++ b/ydb/core/cms/console/console_configs_provider.cpp @@ -801,6 +801,7 @@ void TConfigsProvider::Handle(TEvConsole::TEvConfigSubscriptionRequest::TPtr &ev subscription->NodeId = rec.GetOptions().GetNodeId(); subscription->Host = rec.GetOptions().GetHost(); subscription->Tenant = rec.GetOptions().GetTenant(); + Cerr << " xxx 3 " << subscription->Tenant << " " << rec.GetServeYaml() << Endl; subscription->NodeType = rec.GetOptions().GetNodeType(); subscription->ItemKinds.insert(rec.GetConfigItemKinds().begin(), rec.GetConfigItemKinds().end()); @@ -1224,10 +1225,12 @@ void TConfigsProvider::Handle(TEvPrivate::TEvUpdateSubscriptions::TPtr &ev, cons void TConfigsProvider::Handle(TEvPrivate::TEvUpdateYamlConfig::TPtr &ev, const TActorContext &ctx) { YamlConfig = ev->Get()->YamlConfig; + YamlConfigPerDatabase = ev->Get()->YamlConfigPerDatabase; VolatileYamlConfigs.clear(); YamlConfigVersion = NYamlConfig::GetVersion(YamlConfig); VolatileYamlConfigHashes.clear(); + for (auto& [id, config] : ev->Get()->VolatileYamlConfigs) { auto doc = NFyaml::TDocument::Parse(config); // we strip it to provide old format for config dispatcher @@ -1263,6 +1266,15 @@ void TConfigsProvider::Handle(TEvPrivate::TEvUpdateYamlConfig::TPtr &ev, const T subscription->VolatileYamlConfigHashes = VolatileYamlConfigHashes; + if (YamlConfigPerDatabase.contains(subscription->Tenant)) { + request->Record.SetDatabaseConfig(YamlConfigPerDatabase[subscription->Tenant]); + } else { + Cerr << " xxx search " << subscription->Tenant << Endl; + if (YamlConfigPerDatabase.size() > 0) { + Cerr << " xxx has " << YamlConfigPerDatabase.begin()->first << Endl; + } + } + ctx.Send(subscription->Worker, request.Release()); } } diff --git a/ydb/core/cms/console/console_configs_provider.h b/ydb/core/cms/console/console_configs_provider.h index 7f6c18e466bb..b230d7fa3ddc 100644 --- a/ydb/core/cms/console/console_configs_provider.h +++ b/ydb/core/cms/console/console_configs_provider.h @@ -100,13 +100,24 @@ class TConfigsProvider : public TActorBootstrapped { }; struct TEvUpdateYamlConfig : public TEventLocal { - TEvUpdateYamlConfig(const TString &yamlConfig, const TMap &volatileYamlConfigs = {}) + TEvUpdateYamlConfig( + const TString &yamlConfig, + const TMap &volatileYamlConfigs = {}) : YamlConfig(yamlConfig) , VolatileYamlConfigs(volatileYamlConfigs) { } + TEvUpdateYamlConfig( + const TString &yamlConfig, + const THashMap &yamlConfigPerDatabase) + : YamlConfig(yamlConfig) + , YamlConfigPerDatabase(yamlConfigPerDatabase) + { + } + TString YamlConfig; + THashMap YamlConfigPerDatabase; TMap VolatileYamlConfigs; }; @@ -229,6 +240,7 @@ class TConfigsProvider : public TActorBootstrapped { TString YamlConfig; TMap VolatileYamlConfigs; + THashMap YamlConfigPerDatabase; ui64 YamlConfigVersion = 0; TMap VolatileYamlConfigHashes; }; diff --git a/ydb/core/cms/console/console_configs_subscriber.cpp b/ydb/core/cms/console/console_configs_subscriber.cpp index f69ab7307a87..0d8362f7146f 100644 --- a/ydb/core/cms/console/console_configs_subscriber.cpp +++ b/ydb/core/cms/console/console_configs_subscriber.cpp @@ -220,6 +220,9 @@ class TConfigsSubscriber : public TActorBootstrapped { bool notChanged = true; if (ServeYaml) { + if (rec.HasDatabaseConfig()) { + notChanged = false; + } if (!(rec.HasYamlConfigNotChanged() && rec.GetYamlConfigNotChanged())) { if (rec.HasYamlConfig()) { YamlConfig = rec.GetYamlConfig(); @@ -285,7 +288,8 @@ class TConfigsSubscriber : public TActorBootstrapped { changes, YamlConfig, VolatileYamlConfigs, - CurrentDynConfig), + CurrentDynConfig, + rec.HasDatabaseConfig() ? TMaybe(rec.GetDatabaseConfig()) : TMaybe{}), IEventHandle::FlagTrackDelivery, Cookie); FirstUpdateSent = true; @@ -357,6 +361,7 @@ class TConfigsSubscriber : public TActorBootstrapped { request->Record.SetGeneration(Generation = NextGeneration++); request->Record.MutableOptions()->SetNodeId(SelfId().NodeId()); + Cerr << " xxx 2 " << Tenant << Endl; request->Record.MutableOptions()->SetTenant(Tenant); request->Record.MutableOptions()->SetNodeType(NodeType); request->Record.MutableOptions()->SetHost(FQDNHostName()); diff --git a/ydb/core/grpc_services/rpc_dynamic_config.cpp b/ydb/core/grpc_services/rpc_dynamic_config.cpp index b779e6283d73..2ed2c64578fd 100644 --- a/ydb/core/grpc_services/rpc_dynamic_config.cpp +++ b/ydb/core/grpc_services/rpc_dynamic_config.cpp @@ -206,6 +206,9 @@ class TDynamicConfigRPC : public TRpcOperationRequestActorRecord.MutableRequest()->CopyFrom(*this->GetProtoRequest()); request->Record.SetUserToken(this->Request_->GetSerializedToken()); request->Record.SetPeerName(this->Request_->GetPeerName()); + if (this->Request_->GetDatabaseName()) { + request->Record.SetDatabase(*this->Request_->GetDatabaseName()); + } NTabletPipe::SendData(IActor::SelfId(), ConsolePipe, request.Release()); } }; diff --git a/ydb/core/protos/console_config.proto b/ydb/core/protos/console_config.proto index b161c98de09b..2515842ec7a2 100644 --- a/ydb/core/protos/console_config.proto +++ b/ydb/core/protos/console_config.proto @@ -263,6 +263,7 @@ message TSetYamlConfigRequest { optional Ydb.DynamicConfig.SetConfigRequest Request = 1; optional bytes UserToken = 2; optional string PeerName = 3; + optional string Database = 4; } message TSetYamlConfigResponse { @@ -273,6 +274,7 @@ message TReplaceYamlConfigRequest { optional Ydb.DynamicConfig.ReplaceConfigRequest Request = 1; optional bytes UserToken = 2; optional string PeerName = 3; + optional string Database = 4; } message TReplaceYamlConfigResponse { @@ -283,6 +285,7 @@ message TDropConfigRequest { optional Ydb.DynamicConfig.DropConfigRequest Request = 1; optional bytes UserToken = 2; optional string PeerName = 3; + optional string Database = 4; } message TDropConfigResponse { @@ -292,6 +295,7 @@ message TAddVolatileConfigRequest { optional Ydb.DynamicConfig.AddVolatileConfigRequest Request = 1; optional bytes UserToken = 2; optional string PeerName = 3; + optional string Database = 4; } message TAddVolatileConfigResponse { @@ -301,6 +305,7 @@ message TGetAllConfigsRequest { optional Ydb.DynamicConfig.GetConfigRequest Request = 1; optional bytes UserToken = 2; optional string PeerName = 3; + optional string Database = 4; } message TGetAllConfigsResponse { @@ -311,6 +316,7 @@ message TGetAllConfigsResponse { message TIsYamlReadOnlyRequest { optional bytes UserToken = 1; optional string PeerName = 3; + optional string Database = 4; } message TIsYamlReadOnlyResponse { @@ -321,6 +327,7 @@ message TGetNodeLabelsRequest { optional Ydb.DynamicConfig.GetNodeLabelsRequest Request = 1; optional bytes UserToken = 2; optional string PeerName = 3; + optional string Database = 4; } message TGetNodeLabelsResponse { @@ -331,6 +338,7 @@ message TGetAllMetadataRequest { optional Ydb.DynamicConfig.GetMetadataRequest Request = 1; optional bytes UserToken = 2; optional string PeerName = 3; + optional string Database = 4; } message TGetAllMetadataResponse { @@ -341,6 +349,7 @@ message TRemoveVolatileConfigRequest { optional Ydb.DynamicConfig.RemoveVolatileConfigRequest Request = 1; optional bytes UserToken = 2; optional string PeerName = 3; + optional string Database = 4; } message TRemoveVolatileConfigResponse { @@ -350,6 +359,7 @@ message TResolveConfigRequest { optional Ydb.DynamicConfig.ResolveConfigRequest Request = 1; optional bytes UserToken = 2; optional string PeerName = 3; + optional string Database = 4; } message TResolveConfigResponse { @@ -360,6 +370,7 @@ message TResolveAllConfigRequest { optional Ydb.DynamicConfig.ResolveAllConfigRequest Request = 1; optional bytes UserToken = 2; optional string PeerName = 3; + optional string Database = 4; } message TResolveAllConfigResponse { @@ -503,6 +514,7 @@ message TConfigSubscriptionNotification { optional string YamlConfig = 5; optional bool YamlConfigNotChanged = 6; repeated TVolatileYamlConfig VolatileConfigs = 7; + optional string DatabaseConfig = 8; optional NKikimrConfig.TAppConfig RawConsoleConfig = 103; // Used for correct next startup config prediction } diff --git a/ydb/core/protos/feature_flags.proto b/ydb/core/protos/feature_flags.proto index 8553c96c292e..076d61905031 100644 --- a/ydb/core/protos/feature_flags.proto +++ b/ydb/core/protos/feature_flags.proto @@ -188,4 +188,5 @@ message TFeatureFlags { optional bool EnableTopicTransfer = 163 [default = false]; optional bool EnableViewExport = 164 [default = false]; optional bool EnableColumnStore = 165 [default = false]; + optional bool PerDatabaseConfigAllowed = 166 [default = false]; } diff --git a/ydb/library/yaml_config/public/yaml_config.cpp b/ydb/library/yaml_config/public/yaml_config.cpp index 89ba29283908..d7d5f06f3136 100644 --- a/ydb/library/yaml_config/public/yaml_config.cpp +++ b/ydb/library/yaml_config/public/yaml_config.cpp @@ -742,6 +742,10 @@ bool IsMainConfig(const TString& config) { return IsConfigKindEquals(config, "MainConfig"); } +bool IsDatabaseConfig(const TString& config) { + return IsConfigKindEquals(config, "DatabaseConfig"); +} + TString StripMetadata(const TString& config) { auto doc = NFyaml::TDocument::Parse(config); diff --git a/ydb/library/yaml_config/public/yaml_config.h b/ydb/library/yaml_config/public/yaml_config.h index 157a0acd337f..af66579e3a69 100644 --- a/ydb/library/yaml_config/public/yaml_config.h +++ b/ydb/library/yaml_config/public/yaml_config.h @@ -218,6 +218,11 @@ bool IsVolatileConfig(const TString& config); */ bool IsMainConfig(const TString& config); +/** + * Checks whether string is main config or not + */ +bool IsDatabaseConfig(const TString& config); + /** * Strips metadata from config */ diff --git a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp index d6e2995b8e6b..58d4e0462333 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp @@ -45,6 +45,7 @@ void TCommandConfigFetch::Config(TConfig& config) { .NoArgument().SetFlag(&StripMetadata); config.SetFreeArgsNum(0); + config.AllowEmptyDatabase = true; config.Opts->MutuallyExclusive("all", "strip-metadata"); config.Opts->MutuallyExclusive("output-directory", "strip-metadata"); } @@ -123,6 +124,7 @@ void TCommandConfigReplace::Config(TConfig& config) { .NoArgument().SetFlag(&AllowUnknownFields); config.Opts->AddLongOption("force", "Ignore metadata on config replacement") .NoArgument().SetFlag(&Force); + config.AllowEmptyDatabase = true; config.SetFreeArgsNum(0); } @@ -613,7 +615,6 @@ void TCommandConfigVolatileFetch::Config(TConfig& config) { config.Opts->AddLongOption("strip-metadata", "Strip metadata from config(s)") .NoArgument().SetFlag(&StripMetadata); config.SetFreeArgsNum(0); - config.Opts->MutuallyExclusive("output-directory", "strip-metadata"); } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp b/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp index 58ccceeac9d9..6287ffc455e8 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp @@ -134,8 +134,12 @@ void TClientCommandRootCommon::Config(TConfig& config) { opts.AddLongOption('e', "endpoint", endpointHelp) .RequiredArgument("[PROTOCOL://]HOST[:PORT]").StoreResult(&Address); - opts.AddLongOption('d', "database", databaseHelp) + auto& db = opts.AddLongOption('d', "database", databaseHelp) .RequiredArgument("PATH").StoreResult(&Database); + auto& cluster = opts.AddLongOption('c', "cluster", "Clear database").Optional().NoArgument().Handler0([&](){ + ExplicitNoDatabase = true; + }); + opts.MutuallyExclusiveOpt(db, cluster); opts.AddLongOption('v', "verbose", "Increase verbosity of operations") .Optional().NoArgument().Handler0([&](){ VerbosityLevel++; @@ -510,6 +514,11 @@ void TClientCommandRootCommon::ParseDatabase(TConfig& config) { config.ConnectionParams["database"].push_back({param, sourceText}); return false; }; + + // Priority 0. Explicit --cluster option + if (ExplicitNoDatabase) { + return; + } // Priority 1. Explicit --database option if (Database && getDatabase(Database, "explicit --database option", true)) { return; diff --git a/ydb/public/lib/ydb_cli/commands/ydb_root_common.h b/ydb/public/lib/ydb_cli/commands/ydb_root_common.h index e87a587b2fb8..016a02285970 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_root_common.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_root_common.h @@ -85,6 +85,7 @@ class TClientCommandRootCommon : public TClientCommandRootBase { bool IsIamEndpointSet = false; bool IsCaCertsFileSet = false; bool IsAuthSet = false; + bool ExplicitNoDatabase = false; }; } From 3d1db485fd26a50058fbbee94b1d88b3a823c3d7 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Wed, 15 Jan 2025 17:10:24 +0000 Subject: [PATCH 02/54] WIP --- .../console/console__replace_yaml_config.cpp | 96 +++++++++++++++---- 1 file changed, 80 insertions(+), 16 deletions(-) diff --git a/ydb/core/cms/console/console__replace_yaml_config.cpp b/ydb/core/cms/console/console__replace_yaml_config.cpp index ccd3d935fdcb..6373b86e3280 100644 --- a/ydb/core/cms/console/console__replace_yaml_config.cpp +++ b/ydb/core/cms/console/console__replace_yaml_config.cpp @@ -71,13 +71,93 @@ class TConfigsManager::TTxReplaceYamlConfig : public TTransactionBaseRecord.AddIssues(); + issue->set_severity(errorLevel); + issue->set_message(TStringBuilder{} << "Unknown key# " << info.first << " in proto# " << info.second << " found in path# " << path); + } + + for (auto& [path, info] : result.DeprecatedFields) { + auto *issue = ev->Record.AddIssues(); + issue->set_severity(NYql::TSeverityIds::S_WARNING); + issue->set_message(TStringBuilder{} << "Deprecated key# " << info.first << " in proto# " << info.second << " found in path# " << path); + } + + Response = MakeHolder(Sender, ctx.SelfID, ev.Release()); + }; + + bool isMainConfig = NYamlConfig::IsMainConfig(Config); + bool isDatabaseConfig = NYamlConfig::IsDatabaseConfig(Config); Version = result.Version; UpdatedConfig = result.UpdatedConfig; Cluster = result.Cluster; Modify = result.Modify; + if (!isMainConfig && !isDatabaseConfig) { + Error = true; + auto ev = MakeHolder(); + + auto *issue = ev->Record.AddIssues(); + ErrorReason = "Unknown config kind"; + issue->set_severity(NYql::TSeverityIds::S_ERROR); + issue->set_message(ErrorReason); + ev->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); + Response = MakeHolder(Sender, ctx.SelfID, ev.Release()); + return true; + } + + if (Database && isMainConfig) { + WarnDatabaseByPass = true; + } + + if (Database && isDatabaseConfig && !AppData(ctx)->FeatureFlags.GetPerDatabaseConfigAllowed()) { + Error = true; + auto ev = MakeHolder(); + + auto *issue = ev->Record.AddIssues(); + ErrorReason = "Per database config is disabled"; + issue->set_severity(NYql::TSeverityIds::S_ERROR); + issue->set_message(ErrorReason); + ev->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); + Response = MakeHolder(Sender, ctx.SelfID, ev.Release()); + return true; + } + + if (Database && isDatabaseConfig) { + Self->YamlConfigPerDatabase[*Database] = Config; + // FIXME + Modify = true; + UpdatedConfig = Self->YamlConfig; + // TODO + // 1) send + // 1) persist + // 2) validate + // 3) support force + // 4) support dry-run + // 5) support audit + auto ev = MakeHolder(); + fillResponse(ev, NYql::TSeverityIds::S_WARNING); + + return true; + } + + if (!isMainConfig) { + Error = true; + auto ev = MakeHolder(); + + auto *issue = ev->Record.AddIssues(); + ErrorReason = "Invalid config kind"; + issue->set_severity(NYql::TSeverityIds::S_ERROR); + issue->set_message(ErrorReason); + ev->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); + Response = MakeHolder(Sender, ctx.SelfID, ev.Release()); + return true; + } + if (result.ValidationFinished) { if (!DryRun && !result.HasForbiddenUnknown) { DoInternalAudit(txc, ctx); @@ -95,22 +175,6 @@ class TConfigsManager::TTxReplaceYamlConfig : public TTransactionBaseRecord.AddIssues(); - issue->set_severity(errorLevel); - issue->set_message(TStringBuilder{} << "Unknown key# " << info.first << " in proto# " << info.second << " found in path# " << path); - } - - for (auto& [path, info] : result.DeprecatedFields) { - auto *issue = ev->Record.AddIssues(); - issue->set_severity(NYql::TSeverityIds::S_WARNING); - issue->set_message(TStringBuilder{} << "Deprecated key# " << info.first << " in proto# " << info.second << " found in path# " << path); - } - - Response = MakeHolder(Sender, ctx.SelfID, ev.Release()); - }; - if (result.HasForbiddenUnknown) { Error = true; auto ev = MakeHolder(); From 4022f4bf1987dbd16e30013cf4b1f83915ad92df Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Fri, 10 Jan 2025 21:00:54 +0000 Subject: [PATCH 03/54] distinguish ydb-cli admin commands from common one better better better better better better better --- .../driver_lib/cli_base/cli_cmds_root.cpp | 2 ++ ydb/public/lib/ydb_cli/commands/ydb_admin.cpp | 26 ++++++++++++++++--- .../lib/ydb_cli/commands/ydb_cluster.cpp | 2 ++ .../ydb_cli/commands/ydb_dynamic_config.cpp | 16 +++++++++++- .../lib/ydb_cli/commands/ydb_dynamic_config.h | 5 +++- .../lib/ydb_cli/commands/ydb_root_common.cpp | 22 +++++++++++----- .../lib/ydb_cli/commands/ydb_root_common.h | 1 + .../ydb_cli/commands/ydb_service_scheme.cpp | 24 ++++++++--------- .../ydb_cli/commands/ydb_storage_config.cpp | 15 ++++++++++- .../lib/ydb_cli/commands/ydb_storage_config.h | 5 +++- ydb/public/lib/ydb_cli/commands/ydb_tools.cpp | 2 +- ydb/public/lib/ydb_cli/common/command.cpp | 23 +++++++++++++++- ydb/public/lib/ydb_cli/common/command.h | 26 +++++++++++++++++-- ydb/public/lib/ydb_cli/common/root.cpp | 2 -- 14 files changed, 140 insertions(+), 31 deletions(-) diff --git a/ydb/core/driver_lib/cli_base/cli_cmds_root.cpp b/ydb/core/driver_lib/cli_base/cli_cmds_root.cpp index dc7a92f11ddc..a42c7cb20a7d 100644 --- a/ydb/core/driver_lib/cli_base/cli_cmds_root.cpp +++ b/ydb/core/driver_lib/cli_base/cli_cmds_root.cpp @@ -66,6 +66,8 @@ void TClientCommandRootKikimrBase::Parse(TConfig& config) { ParseProfile(); GetProfileVariable("path", config.Path); TClientCommandRootBase::Parse(config); + ParseCredentials(config); + ParseAddress(config); NClient::TKikimr::DUMP_REQUESTS = DumpRequests; } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_admin.cpp b/ydb/public/lib/ydb_cli/commands/ydb_admin.cpp index 14f263455ea3..c828bc21ee58 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_admin.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_admin.cpp @@ -7,13 +7,33 @@ namespace NYdb { namespace NConsoleClient { +class TCommandNode : public TClientCommandTree { +public: + TCommandNode() + : TClientCommandTree("node", {}, "Node-wide administration") + {} +}; + +class TCommandDatabase : public TClientCommandTree { +public: + TCommandDatabase() + : TClientCommandTree("database", {}, "Database-wide administration") + { + AddCommand(std::make_unique()); + } +}; + TCommandAdmin::TCommandAdmin() : TClientCommandTree("admin", {}, "Administrative cluster operations") { - AddCommand(std::make_unique()); - AddCommand(std::make_unique()); - AddCommand(std::make_unique()); + MarkDangerous(); + UseOnlyExplicitProfile(); + AddHiddenCommand(std::make_unique(false)); + AddHiddenCommand(std::make_unique()); + AddHiddenCommand(std::make_unique(false)); AddCommand(std::make_unique()); + AddCommand(std::make_unique()); + AddCommand(std::make_unique()); } } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp b/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp index 78db85a9009d..1b65bc92c764 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp @@ -1,6 +1,7 @@ #include "ydb_cluster.h" #include +#include "ydb_dynamic_config.h" using namespace NKikimr; @@ -10,6 +11,7 @@ TCommandCluster::TCommandCluster() : TClientCommandTree("cluster", {}, "Cluster-wide administration") { AddCommand(std::make_unique()); + AddCommand(std::make_unique()); } TCommandClusterBootstrap::TCommandClusterBootstrap() diff --git a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp index 58d4e0462333..5eaaee9d9a39 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp @@ -22,14 +22,28 @@ TString WrapYaml(const TString& yaml) { return out.Str(); } -TCommandConfig::TCommandConfig() +TCommandConfig::TCommandConfig(std::optional overrideOnlyExplicitProfile) : TClientCommandTree("config", {}, "Dynamic config") + , OverrideOnlyExplicitProfile(overrideOnlyExplicitProfile) { AddCommand(std::make_unique()); AddCommand(std::make_unique()); AddCommand(std::make_unique()); } + +void TCommandConfig::PropagateFlags(const TCommandFlags& flags) { + TClientCommand::PropagateFlags(flags); + + if (OverrideOnlyExplicitProfile) { + OnlyExplicitProfile = *OverrideOnlyExplicitProfile; + } + + for (auto& [_, cmd] : SubCommands) { + cmd->PropagateFlags(TCommandFlags{.Dangerous = Dangerous, .OnlyExplicitProfile = OnlyExplicitProfile}); + } +} + TCommandConfigFetch::TCommandConfigFetch() : TYdbCommand("fetch", {"get", "dump"}, "Fetch main dynamic-config") { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.h b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.h index 81e6748fa948..4508af7bff08 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.h @@ -9,7 +9,10 @@ namespace NYdb::NConsoleClient::NDynamicConfig { class TCommandConfig : public TClientCommandTree { public: - TCommandConfig(); + TCommandConfig(std::optional overrideOnlyExplicitProfile = std::nullopt); + void PropagateFlags(const TCommandFlags& flags) override; +private: + std::optional OverrideOnlyExplicitProfile; }; class TCommandConfigReplace : public TYdbCommand { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp b/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp index 6287ffc455e8..b01db54f7bed 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp @@ -36,7 +36,7 @@ TClientCommandRootCommon::TClientCommandRootCommon(const TString& name, const TC , Settings(settings) { ValidateSettings(); - AddCommand(std::make_unique()); + AddDangerousCommand(std::make_unique()); AddCommand(std::make_unique()); AddCommand(std::make_unique()); AddCommand(std::make_unique()); @@ -54,6 +54,7 @@ TClientCommandRootCommon::TClientCommandRootCommon(const TString& name, const TC AddCommand(std::make_unique()); AddCommand(std::make_unique()); AddCommand(std::make_unique()); + PropagateFlags(TCommandFlags{.Dangerous = false, .OnlyExplicitProfile = false}); } void TClientCommandRootCommon::ValidateSettings() { @@ -332,6 +333,11 @@ void TClientCommandRootCommon::Config(TConfig& config) { } void TClientCommandRootCommon::Parse(TConfig& config) { + TClientCommandRootBase::Parse(config); + config.VerbosityLevel = std::min(static_cast(VerbosityLevel), TConfig::EVerbosityLevel::DEBUG); +} + +void TClientCommandRootCommon::PostPrepare(TConfig& config) { if (ProfileFile.empty()) { config.ProfileFile = TStringBuilder() << HomeDir << '/' << Settings.YdbDir << "/config/config.yaml"; } else { @@ -343,12 +349,12 @@ void TClientCommandRootCommon::Parse(TConfig& config) { ProfileManager = CreateProfileManager(config.ProfileFile); ParseProfile(); - TClientCommandRootBase::Parse(config); ParseDatabase(config); ParseCaCerts(config); ParseIamEndpoint(config); - config.VerbosityLevel = std::min(static_cast(VerbosityLevel), TConfig::EVerbosityLevel::DEBUG); + ParseCredentials(config); + ParseAddress(config); } namespace { @@ -439,7 +445,7 @@ void TClientCommandRootCommon::ParseAddress(TConfig& config) { return; } // Priority 3. Active profile (if --profile option is not specified) - if (TryGetParamFromProfile("endpoint", ProfileManager->GetActiveProfile(), false, getAddress)) { + if (!config.OnlyExplicitProfile && TryGetParamFromProfile("endpoint", ProfileManager->GetActiveProfile(), false, getAddress)) { return; } } @@ -528,7 +534,7 @@ void TClientCommandRootCommon::ParseDatabase(TConfig& config) { return; } // Priority 3. Active profile (if --profile option is not specified) - if (TryGetParamFromProfile("database", ProfileManager->GetActiveProfile(), false, getDatabase)) { + if (!config.OnlyExplicitProfile && TryGetParamFromProfile("database", ProfileManager->GetActiveProfile(), false, getDatabase)) { return; } } @@ -583,11 +589,15 @@ void TClientCommandRootCommon::Validate(TConfig& config) { throw TMisuseException() << errors; } } + + // TODO: Maybe NeedToConnect doesn't always mean that we don't need to check endpoint and database + // TODO: Now we supplying only one error while it is possible to return all errors at once, + // maybe even erros from nested command's validate if (!config.NeedToConnect) { return; } - if (config.Address.empty()) { + if (config.Address.empty() && config.AllowEmptyAddress) { throw TMisuseException() << "Missing required option 'endpoint'."; } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_root_common.h b/ydb/public/lib/ydb_cli/commands/ydb_root_common.h index 016a02285970..bd2d2d7545ef 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_root_common.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_root_common.h @@ -31,6 +31,7 @@ class TClientCommandRootCommon : public TClientCommandRootBase { public: TClientCommandRootCommon(const TString& name, const TClientSettings& settings); void Config(TConfig& config) override; + void PostPrepare(TConfig& config) override; void Parse(TConfig& config) override; void ParseAddress(TConfig& config) override; void ParseCredentials(TConfig& config) override; diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp index 75627fbe119a..a2cfa66dc4c2 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp @@ -43,10 +43,10 @@ void TCommandMakeDirectory::Config(TConfig& config) { void TCommandMakeDirectory::Parse(TConfig& config) { TClientCommand::Parse(config); - ParsePath(config, 0); } int TCommandMakeDirectory::Run(TConfig& config) { + ParsePath(config, 0); NScheme::TSchemeClient client(CreateDriver(config)); NStatusHelpers::ThrowOnErrorOrPrintIssues( client.MakeDirectory( @@ -78,10 +78,10 @@ void TCommandRemoveDirectory::Config(TConfig& config) { void TCommandRemoveDirectory::Parse(TConfig& config) { TClientCommand::Parse(config); - ParsePath(config, 0); } int TCommandRemoveDirectory::Run(TConfig& config) { + ParsePath(config, 0); TDriver driver = CreateDriver(config); NScheme::TSchemeClient schemeClient(driver); const auto settings = FillSettings(NScheme::TRemoveDirectorySettings()); @@ -245,10 +245,10 @@ void TCommandDescribe::Parse(TConfig& config) { TClientCommand::Parse(config); Database = config.Database; ParseOutputFormats(); - ParsePath(config, 0); } int TCommandDescribe::Run(TConfig& config) { + ParsePath(config, 0); TDriver driver = CreateDriver(config); NScheme::TSchemeClient client(driver); NScheme::TDescribePathResult result = client.DescribePath( @@ -1059,7 +1059,6 @@ void TCommandList::Config(TConfig& config) { void TCommandList::Parse(TConfig& config) { TClientCommand::Parse(config); - ParsePath(config, 0, true); if (AdvancedMode && FromNewLine) { // TODO: add "consider using --format shell" throw TMisuseException() << "Options -1 and -l are incompatible"; @@ -1067,6 +1066,7 @@ void TCommandList::Parse(TConfig& config) { } int TCommandList::Run(TConfig& config) { + ParsePath(config, 0, true); TDriver driver = CreateDriver(config); ISchemePrinter::TSettings settings = { Path, @@ -1129,7 +1129,6 @@ void TCommandPermissionGrant::Config(TConfig& config) { void TCommandPermissionGrant::Parse(TConfig& config) { TClientCommand::Parse(config); - ParsePath(config, 0); Subject = config.ParseResult->GetFreeArgs()[1]; if (Subject.empty()) { throw TMisuseException() << "Missing required argument "; @@ -1140,6 +1139,7 @@ void TCommandPermissionGrant::Parse(TConfig& config) { } int TCommandPermissionGrant::Run(TConfig& config) { + ParsePath(config, 0); NScheme::TSchemeClient client(CreateDriver(config)); NStatusHelpers::ThrowOnErrorOrPrintIssues( client.ModifyPermissions( @@ -1170,7 +1170,6 @@ void TCommandPermissionRevoke::Config(TConfig& config) { void TCommandPermissionRevoke::Parse(TConfig& config) { TClientCommand::Parse(config); - ParsePath(config, 0); Subject = config.ParseResult->GetFreeArgs()[1]; if (Subject.empty()) { throw TMisuseException() << "Missing required argument "; @@ -1181,6 +1180,7 @@ void TCommandPermissionRevoke::Parse(TConfig& config) { } int TCommandPermissionRevoke::Run(TConfig& config) { + ParsePath(config, 0); NScheme::TSchemeClient client(CreateDriver(config)); NStatusHelpers::ThrowOnErrorOrPrintIssues( client.ModifyPermissions( @@ -1211,7 +1211,6 @@ void TCommandPermissionSet::Config(TConfig& config) { void TCommandPermissionSet::Parse(TConfig& config) { TClientCommand::Parse(config); - ParsePath(config, 0); Subject = config.ParseResult->GetFreeArgs()[1]; if (Subject.empty()) { throw TMisuseException() << "Missing required argument "; @@ -1222,6 +1221,7 @@ void TCommandPermissionSet::Parse(TConfig& config) { } int TCommandPermissionSet::Run(TConfig& config) { + ParsePath(config, 0); NScheme::TSchemeClient client(CreateDriver(config)); NStatusHelpers::ThrowOnErrorOrPrintIssues( client.ModifyPermissions( @@ -1249,7 +1249,6 @@ void TCommandChangeOwner::Config(TConfig& config) { void TCommandChangeOwner::Parse(TConfig& config) { TClientCommand::Parse(config); - ParsePath(config, 0); Owner = config.ParseResult->GetFreeArgs()[1]; if (!Owner){ throw TMisuseException() << "Missing required argument "; @@ -1257,6 +1256,7 @@ void TCommandChangeOwner::Parse(TConfig& config) { } int TCommandChangeOwner::Run(TConfig& config) { + ParsePath(config, 0); NScheme::TSchemeClient client(CreateDriver(config)); NStatusHelpers::ThrowOnErrorOrPrintIssues( client.ModifyPermissions( @@ -1283,10 +1283,10 @@ void TCommandPermissionClear::Config(TConfig& config) { void TCommandPermissionClear::Parse(TConfig& config) { TClientCommand::Parse(config); - ParsePath(config, 0); } int TCommandPermissionClear::Run(TConfig& config) { + ParsePath(config, 0); NScheme::TSchemeClient client(CreateDriver(config)); NStatusHelpers::ThrowOnErrorOrPrintIssues( client.ModifyPermissions( @@ -1313,10 +1313,10 @@ void TCommandPermissionSetInheritance::Config(TConfig& config) { void TCommandPermissionSetInheritance::Parse(TConfig& config) { TClientCommand::Parse(config); - ParsePath(config, 0); } int TCommandPermissionSetInheritance::Run(TConfig& config) { + ParsePath(config, 0); NScheme::TSchemeClient client(CreateDriver(config)); NStatusHelpers::ThrowOnErrorOrPrintIssues( client.ModifyPermissions( @@ -1343,10 +1343,10 @@ void TCommandPermissionClearInheritance::Config(TConfig& config) { void TCommandPermissionClearInheritance::Parse(TConfig& config) { TClientCommand::Parse(config); - ParsePath(config, 0); } int TCommandPermissionClearInheritance::Run(TConfig& config) { + ParsePath(config, 0); NScheme::TSchemeClient client(CreateDriver(config)); NStatusHelpers::ThrowOnErrorOrPrintIssues( client.ModifyPermissions( @@ -1373,10 +1373,10 @@ void TCommandPermissionList::Config(TConfig& config) { void TCommandPermissionList::Parse(TConfig& config) { TClientCommand::Parse(config); - ParsePath(config, 0); } int TCommandPermissionList::Run(TConfig& config) { + ParsePath(config, 0); TDriver driver = CreateDriver(config); NScheme::TSchemeClient client(driver); NScheme::TDescribePathResult result = client.DescribePath( diff --git a/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp b/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp index 87f17b751a2c..5fb2cfd72c04 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp @@ -22,13 +22,26 @@ TString WrapYaml(const TString& yaml) { return out.Str(); } -TCommandStorageConfig::TCommandStorageConfig() +TCommandStorageConfig::TCommandStorageConfig(std::optional overrideOnlyExplicitProfile) : TClientCommandTree("storage", {}, "Storage config") + , OverrideOnlyExplicitProfile(overrideOnlyExplicitProfile) { AddCommand(std::make_unique()); AddCommand(std::make_unique()); } +void TCommandStorageConfig::PropagateFlags(const TCommandFlags& flags) { + TClientCommand::PropagateFlags(flags); + + if (OverrideOnlyExplicitProfile) { + OnlyExplicitProfile = *OverrideOnlyExplicitProfile; + } + + for (auto& [_, cmd] : SubCommands) { + cmd->PropagateFlags(TCommandFlags{.Dangerous = Dangerous, .OnlyExplicitProfile = OnlyExplicitProfile}); + } +} + TCommandStorageConfigFetch::TCommandStorageConfigFetch() : TYdbCommand("fetch", {}, "Fetch storage config") { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_storage_config.h b/ydb/public/lib/ydb_cli/commands/ydb_storage_config.h index a5c440a569ee..1092824e0e26 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_storage_config.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_storage_config.h @@ -9,7 +9,10 @@ namespace NYdb::NConsoleClient::NStorageConfig { class TCommandStorageConfig : public TClientCommandTree { public: - TCommandStorageConfig(); + TCommandStorageConfig(std::optional overrideOnlyExplicitProfile = std::nullopt); + void PropagateFlags(const TCommandFlags& flags) override; +private: + std::optional OverrideOnlyExplicitProfile; }; class TCommandStorageConfigReplace : public TYdbCommand { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_tools.cpp b/ydb/public/lib/ydb_cli/commands/ydb_tools.cpp index c24e8c639030..3821105587a2 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_tools.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_tools.cpp @@ -86,10 +86,10 @@ void TCommandDump::Config(TConfig& config) { void TCommandDump::Parse(TConfig& config) { TClientCommand::Parse(config); - AdjustPath(config); } int TCommandDump::Run(TConfig& config) { + AdjustPath(config); NDump::TDumpSettings::EConsistencyLevel consistencyLevel; if (!TryFromString(ConsistencyLevel, consistencyLevel)) { throw yexception() << "Incorrect consistency level." diff --git a/ydb/public/lib/ydb_cli/common/command.cpp b/ydb/public/lib/ydb_cli/common/command.cpp index c4f8dde9e5f5..12e5fb0a0362 100644 --- a/ydb/public/lib/ydb_cli/common/command.cpp +++ b/ydb/public/lib/ydb_cli/common/command.cpp @@ -208,6 +208,7 @@ void TClientCommand::CheckForExecutableOptions(TConfig& config) { void TClientCommand::Config(TConfig& config) { config.Opts = &Opts; + config.OnlyExplicitProfile = OnlyExplicitProfile; TStringStream stream; NColorizer::TColors colors = NColorizer::AutoColors(Cout); stream << Endl << Endl @@ -251,9 +252,14 @@ void TClientCommand::Prepare(TConfig& config) { Parse(config); } +void TClientCommand::PostPrepare(TConfig& config) { + Y_UNUSED(config); +} + int TClientCommand::ValidateAndRun(TConfig& config) { config.Opts = &Opts; config.ParseResult = ParseResult.get(); + PostPrepare(config); Validate(config); return Run(config); } @@ -316,7 +322,7 @@ void TClientCommand::RenderOneCommandDescription( prefix = "└─ "; } TString line = prefix + Name; - stream << prefix << colors.BoldColor() << Name << colors.OldColor(); + stream << prefix << (Dangerous ? colors.Red() : "") << colors.BoldColor() << Name << colors.OldColor(); if (!Description.empty()) { int namePartLength = GetNumberOfUTF8Chars(line); if (namePartLength < DESCRIPTION_ALIGNMENT) @@ -339,6 +345,15 @@ void TClientCommand::RenderOneCommandDescription( void TClientCommand::Hide() { Hidden = true; + Visible = false; +} + +void TClientCommand::MarkDangerous() { + Dangerous = true; +} + +void TClientCommand::UseOnlyExplicitProfile() { + OnlyExplicitProfile = true; } TClientCommandTree::TClientCommandTree(const TString& name, const std::initializer_list& aliases, const TString& description) @@ -361,6 +376,12 @@ void TClientCommandTree::AddHiddenCommand(std::unique_ptr comman AddCommand(std::move(command)); } +void TClientCommandTree::AddDangerousCommand(std::unique_ptr command) { + command->MarkDangerous(); + command->UseOnlyExplicitProfile(); + AddCommand(std::move(command)); +} + void TClientCommandTree::Config(TConfig& config) { TClientCommand::Config(config); SetFreeArgs(config); diff --git a/ydb/public/lib/ydb_cli/common/command.h b/ydb/public/lib/ydb_cli/common/command.h index c1e474275be2..199e0db1ffde 100644 --- a/ydb/public/lib/ydb_cli/common/command.h +++ b/ydb/public/lib/ydb_cli/common/command.h @@ -17,6 +17,11 @@ namespace NYdb { namespace NConsoleClient { +struct TCommandFlags { + bool Dangerous = false; + bool OnlyExplicitProfile = false; +}; + class TClientCommand { public: static bool TIME_REQUESTS; // measure time of requests @@ -25,6 +30,9 @@ class TClientCommand { TVector Aliases; TString Description; bool Visible = true; + bool Hidden = false; + bool Dangerous = false; + bool OnlyExplicitProfile = false; const TClientCommand* Parent; NLastGetopt::TOpts Opts; TString Argument; @@ -137,6 +145,8 @@ class TClientCommand { bool NeedToCheckForUpdate = true; bool ForceVersionCheck = false; bool AllowEmptyDatabase = false; + bool AllowEmptyAddress = false; + bool OnlyExplicitProfile = false; TCredentialsGetter CredentialsGetter; @@ -307,7 +317,12 @@ class TClientCommand { virtual int Process(TConfig& config); virtual void Prepare(TConfig& config); + virtual void PostPrepare(TConfig& config); virtual int ValidateAndRun(TConfig& config); + virtual void PropagateFlags(const TCommandFlags& flags) { + Dangerous |= flags.Dangerous; + OnlyExplicitProfile |= flags.OnlyExplicitProfile; + } enum RenderEntryType { BEGIN, @@ -322,6 +337,8 @@ class TClientCommand { ); void Hide(); + void MarkDangerous(); + void UseOnlyExplicitProfile(); protected: virtual void Config(TConfig& config); @@ -342,7 +359,6 @@ class TClientCommand { void CheckForExecutableOptions(TConfig& config); constexpr static int DESCRIPTION_ALIGNMENT = 28; - bool Hidden = false; }; class TClientCommandTree : public TClientCommand { @@ -350,6 +366,7 @@ class TClientCommandTree : public TClientCommand { TClientCommandTree(const TString& name, const std::initializer_list& aliases = std::initializer_list(), const TString& description = TString()); void AddCommand(std::unique_ptr command); void AddHiddenCommand(std::unique_ptr command); + void AddDangerousCommand(std::unique_ptr command); virtual void Prepare(TConfig& config) override; void RenderCommandsDescription( TStringStream& stream, @@ -363,10 +380,15 @@ class TClientCommandTree : public TClientCommand { virtual void SaveParseResult(TConfig& config) override; virtual void Parse(TConfig& config) override; virtual int Run(TConfig& config) override; + virtual void PropagateFlags(const TCommandFlags& flags) override { + TClientCommand::PropagateFlags(flags); + for (auto& [_, cmd] : SubCommands) { + cmd->PropagateFlags(TCommandFlags{.Dangerous = Dangerous, .OnlyExplicitProfile = OnlyExplicitProfile}); + } + } TClientCommand* SelectedCommand; -private: bool HasOptionsToShow(); TMap> SubCommands; diff --git a/ydb/public/lib/ydb_cli/common/root.cpp b/ydb/public/lib/ydb_cli/common/root.cpp index 98ab7bc1ccd9..3ebbbd2686e7 100644 --- a/ydb/public/lib/ydb_cli/common/root.cpp +++ b/ydb/public/lib/ydb_cli/common/root.cpp @@ -37,8 +37,6 @@ void TClientCommandRootBase::SetCustomUsage(TConfig& config) { void TClientCommandRootBase::Parse(TConfig& config) { TClientCommandTree::Parse(config); - ParseCredentials(config); - ParseAddress(config); TClientCommand::TIME_REQUESTS = TimeRequests; TClientCommand::PROGRESS_REQUESTS = ProgressRequests; From d127585846cea47628ff7532e72f2abef2b30c7b Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Wed, 15 Jan 2025 18:26:55 +0000 Subject: [PATCH 04/54] tmp --- ydb/core/cms/console/console_configs_manager.cpp | 8 ++++++++ ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp | 10 +--------- ydb/public/lib/ydb_cli/commands/ydb_root_common.h | 1 - 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/ydb/core/cms/console/console_configs_manager.cpp b/ydb/core/cms/console/console_configs_manager.cpp index bd572718568b..95890810aa9f 100644 --- a/ydb/core/cms/console/console_configs_manager.cpp +++ b/ydb/core/cms/console/console_configs_manager.cpp @@ -54,6 +54,14 @@ bool TConfigsManager::CheckConfig(const NKikimrConsole::TConfigsConfig &config, TConfigsManager::TValidateConfigResult TConfigsManager::ValidateConfigAndReplaceMetadata(const TString &config, bool force, bool allowUnknownFields) { TValidateConfigResult result; + bool isDatabaseConfig = NYamlConfig::IsDatabaseConfig(config); + + if (isDatabaseConfig) { + result.HasForbiddenUnknown = false; + result.ValidationFinished = true; + return result; + } + try { if (!force) { auto metadata = NYamlConfig::GetMetadata(config); diff --git a/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp b/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp index b01db54f7bed..dbd200041954 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp @@ -135,12 +135,8 @@ void TClientCommandRootCommon::Config(TConfig& config) { opts.AddLongOption('e', "endpoint", endpointHelp) .RequiredArgument("[PROTOCOL://]HOST[:PORT]").StoreResult(&Address); - auto& db = opts.AddLongOption('d', "database", databaseHelp) + opts.AddLongOption('d', "database", databaseHelp) .RequiredArgument("PATH").StoreResult(&Database); - auto& cluster = opts.AddLongOption('c', "cluster", "Clear database").Optional().NoArgument().Handler0([&](){ - ExplicitNoDatabase = true; - }); - opts.MutuallyExclusiveOpt(db, cluster); opts.AddLongOption('v', "verbose", "Increase verbosity of operations") .Optional().NoArgument().Handler0([&](){ VerbosityLevel++; @@ -521,10 +517,6 @@ void TClientCommandRootCommon::ParseDatabase(TConfig& config) { return false; }; - // Priority 0. Explicit --cluster option - if (ExplicitNoDatabase) { - return; - } // Priority 1. Explicit --database option if (Database && getDatabase(Database, "explicit --database option", true)) { return; diff --git a/ydb/public/lib/ydb_cli/commands/ydb_root_common.h b/ydb/public/lib/ydb_cli/commands/ydb_root_common.h index bd2d2d7545ef..cfa571b41d04 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_root_common.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_root_common.h @@ -86,7 +86,6 @@ class TClientCommandRootCommon : public TClientCommandRootBase { bool IsIamEndpointSet = false; bool IsCaCertsFileSet = false; bool IsAuthSet = false; - bool ExplicitNoDatabase = false; }; } From e76eec2968c7495467e59146eb437ef5c783b773 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Wed, 15 Jan 2025 18:30:38 +0000 Subject: [PATCH 05/54] cleanup --- ydb/core/cms/console/configs_dispatcher.cpp | 2 -- ydb/core/cms/console/console_configs_provider.cpp | 6 ------ 2 files changed, 8 deletions(-) diff --git a/ydb/core/cms/console/configs_dispatcher.cpp b/ydb/core/cms/console/configs_dispatcher.cpp index 5c622b114f88..72f2dee849a3 100644 --- a/ydb/core/cms/console/configs_dispatcher.cpp +++ b/ydb/core/cms/console/configs_dispatcher.cpp @@ -891,8 +891,6 @@ void TConfigsDispatcher::Handle(TEvConsole::TEvConfigSubscriptionNotification::T } newYamlConfig.append(newname); - Cerr << " xxx 4 " << rec.GetDatabaseConfig(); - Cerr << " xxx 5 " << newYamlConfig; } if (rec.VolatileConfigsSize() != VolatileYamlConfigs.size()) { diff --git a/ydb/core/cms/console/console_configs_provider.cpp b/ydb/core/cms/console/console_configs_provider.cpp index 787c9db335e7..216f3afb6fe3 100644 --- a/ydb/core/cms/console/console_configs_provider.cpp +++ b/ydb/core/cms/console/console_configs_provider.cpp @@ -801,7 +801,6 @@ void TConfigsProvider::Handle(TEvConsole::TEvConfigSubscriptionRequest::TPtr &ev subscription->NodeId = rec.GetOptions().GetNodeId(); subscription->Host = rec.GetOptions().GetHost(); subscription->Tenant = rec.GetOptions().GetTenant(); - Cerr << " xxx 3 " << subscription->Tenant << " " << rec.GetServeYaml() << Endl; subscription->NodeType = rec.GetOptions().GetNodeType(); subscription->ItemKinds.insert(rec.GetConfigItemKinds().begin(), rec.GetConfigItemKinds().end()); @@ -1268,11 +1267,6 @@ void TConfigsProvider::Handle(TEvPrivate::TEvUpdateYamlConfig::TPtr &ev, const T if (YamlConfigPerDatabase.contains(subscription->Tenant)) { request->Record.SetDatabaseConfig(YamlConfigPerDatabase[subscription->Tenant]); - } else { - Cerr << " xxx search " << subscription->Tenant << Endl; - if (YamlConfigPerDatabase.size() > 0) { - Cerr << " xxx has " << YamlConfigPerDatabase.begin()->first << Endl; - } } ctx.Send(subscription->Worker, request.Release()); From fc1bd99e08b3e240bd0d811bb3064026f0bfa2dc Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Thu, 16 Jan 2025 22:10:01 +0000 Subject: [PATCH 06/54] better --- ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp | 2 +- .../lib/ydb_cli/commands/ydb_dynamic_config.cpp | 16 +++++++++------- .../lib/ydb_cli/commands/ydb_dynamic_config.h | 8 +++++--- .../lib/ydb_cli/commands/ydb_root_common.cpp | 2 +- ydb/public/lib/ydb_cli/common/command.cpp | 2 +- 5 files changed, 17 insertions(+), 13 deletions(-) diff --git a/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp b/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp index 1b65bc92c764..fd4a24129cd2 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp @@ -11,7 +11,7 @@ TCommandCluster::TCommandCluster() : TClientCommandTree("cluster", {}, "Cluster-wide administration") { AddCommand(std::make_unique()); - AddCommand(std::make_unique()); + AddCommand(std::make_unique(std::nullopt, true)); } TCommandClusterBootstrap::TCommandClusterBootstrap() diff --git a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp index 5eaaee9d9a39..ec97c95a657f 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp @@ -22,12 +22,12 @@ TString WrapYaml(const TString& yaml) { return out.Str(); } -TCommandConfig::TCommandConfig(std::optional overrideOnlyExplicitProfile) +TCommandConfig::TCommandConfig(std::optional overrideOnlyExplicitProfile, bool allowEmptyDatabase) : TClientCommandTree("config", {}, "Dynamic config") , OverrideOnlyExplicitProfile(overrideOnlyExplicitProfile) { - AddCommand(std::make_unique()); - AddCommand(std::make_unique()); + AddCommand(std::make_unique(allowEmptyDatabase)); + AddCommand(std::make_unique(allowEmptyDatabase)); AddCommand(std::make_unique()); } @@ -44,8 +44,9 @@ void TCommandConfig::PropagateFlags(const TCommandFlags& flags) { } } -TCommandConfigFetch::TCommandConfigFetch() +TCommandConfigFetch::TCommandConfigFetch(bool allowEmptyDatabase) : TYdbCommand("fetch", {"get", "dump"}, "Fetch main dynamic-config") + , AllowEmptyDatabase(allowEmptyDatabase) { } @@ -59,7 +60,7 @@ void TCommandConfigFetch::Config(TConfig& config) { .NoArgument().SetFlag(&StripMetadata); config.SetFreeArgsNum(0); - config.AllowEmptyDatabase = true; + config.AllowEmptyDatabase = AllowEmptyDatabase; config.Opts->MutuallyExclusive("all", "strip-metadata"); config.Opts->MutuallyExclusive("output-directory", "strip-metadata"); } @@ -120,9 +121,10 @@ int TCommandConfigFetch::Run(TConfig& config) { return EXIT_SUCCESS; } -TCommandConfigReplace::TCommandConfigReplace() +TCommandConfigReplace::TCommandConfigReplace(bool allowEmptyDatabase) : TYdbCommand("replace", {}, "Replace dynamic config") , IgnoreCheck(false) + , AllowEmptyDatabase(allowEmptyDatabase) { } @@ -138,7 +140,7 @@ void TCommandConfigReplace::Config(TConfig& config) { .NoArgument().SetFlag(&AllowUnknownFields); config.Opts->AddLongOption("force", "Ignore metadata on config replacement") .NoArgument().SetFlag(&Force); - config.AllowEmptyDatabase = true; + config.AllowEmptyDatabase = AllowEmptyDatabase; config.SetFreeArgsNum(0); } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.h b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.h index 4508af7bff08..c5bade6f6235 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.h @@ -9,7 +9,7 @@ namespace NYdb::NConsoleClient::NDynamicConfig { class TCommandConfig : public TClientCommandTree { public: - TCommandConfig(std::optional overrideOnlyExplicitProfile = std::nullopt); + TCommandConfig(std::optional overrideOnlyExplicitProfile = std::nullopt, bool allowEmptyDatabase = false); void PropagateFlags(const TCommandFlags& flags) override; private: std::optional OverrideOnlyExplicitProfile; @@ -17,7 +17,7 @@ class TCommandConfig : public TClientCommandTree { class TCommandConfigReplace : public TYdbCommand { public: - TCommandConfigReplace(); + TCommandConfigReplace(bool allowEmptyDatabase); void Config(TConfig& config) override; void Parse(TConfig& config) override; int Run(TConfig& config) override; @@ -29,11 +29,12 @@ class TCommandConfigReplace : public TYdbCommand { bool AllowUnknownFields = false; TString DynamicConfig; TString Filename; + bool AllowEmptyDatabase = false; }; class TCommandConfigFetch : public TYdbCommand { public: - TCommandConfigFetch(); + TCommandConfigFetch(bool allowEmptyDatabase); void Config(TConfig&) override; void Parse(TConfig&) override; int Run(TConfig& config) override; @@ -42,6 +43,7 @@ class TCommandConfigFetch : public TYdbCommand { bool All = false; bool StripMetadata = false; TString OutDir; + bool AllowEmptyDatabase = false; }; class TCommandConfigResolve : public TYdbCommand { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp b/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp index dbd200041954..f03c95cfe078 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp @@ -589,7 +589,7 @@ void TClientCommandRootCommon::Validate(TConfig& config) { return; } - if (config.Address.empty() && config.AllowEmptyAddress) { + if (config.Address.empty() && !config.AllowEmptyAddress) { throw TMisuseException() << "Missing required option 'endpoint'."; } diff --git a/ydb/public/lib/ydb_cli/common/command.cpp b/ydb/public/lib/ydb_cli/common/command.cpp index 12e5fb0a0362..bdad74afb416 100644 --- a/ydb/public/lib/ydb_cli/common/command.cpp +++ b/ydb/public/lib/ydb_cli/common/command.cpp @@ -311,7 +311,7 @@ void TClientCommand::RenderOneCommandDescription( const NColorizer::TColors& colors, RenderEntryType type ) { - if (Hidden) { + if (Hidden && type != BEGIN) { return; } TString prefix; From 295f580dc1de2ad36152b3ad4f09b1c3f9e57974 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Thu, 16 Jan 2025 23:18:31 +0000 Subject: [PATCH 07/54] better --- ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp b/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp index f03c95cfe078..8072d25bdcc9 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp @@ -320,6 +320,7 @@ void TClientCommandRootCommon::Config(TConfig& config) { stream << " [options...] " << Endl << Endl << colors.BoldColor() << "Subcommands" << colors.OldColor() << ":" << Endl; RenderCommandsDescription(stream, colors); + stream << Endl << Endl << colors.BoldColor() << "Commands in " << colors.Red() << colors.BoldColor() << "admin" << colors.OldColor() << colors.BoldColor() << " subtree may treat global flags and profile differently, see corresponding help" << colors.OldColor() << Endl; opts.SetCmdLineDescr(stream.Str()); opts.GetLongOption("time").Hidden(); @@ -590,14 +591,14 @@ void TClientCommandRootCommon::Validate(TConfig& config) { } if (config.Address.empty() && !config.AllowEmptyAddress) { - throw TMisuseException() << "Missing required option 'endpoint'."; + throw TMisuseException() << "Missing required option 'endpoint'." << (config.OnlyExplicitProfile ? " Profile ignored due to admin command use." : ""); } if (config.Database.empty() && config.AllowEmptyDatabase) { // just skip the Database check } else if (config.Database.empty()) { throw TMisuseException() - << "Missing required option 'database'."; + << "Missing required option 'database'." << (config.OnlyExplicitProfile ? " Profile ignored due to admin command use." : ""); } else if (!config.Database.StartsWith('/')) { throw TMisuseException() << "Path to a database \"" << config.Database << "\" is incorrect. It must be absolute and thus must begin with '/'."; From 3e1a5f7d7959b8c430f82b405fa88aa95a2808ec Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Thu, 16 Jan 2025 23:46:08 +0000 Subject: [PATCH 08/54] better --- ydb/public/lib/ydb_cli/commands/ydb_admin.cpp | 108 ++++++++++++++++++ ydb/public/lib/ydb_cli/commands/ydb_admin.h | 2 + 2 files changed, 110 insertions(+) diff --git a/ydb/public/lib/ydb_cli/commands/ydb_admin.cpp b/ydb/public/lib/ydb_cli/commands/ydb_admin.cpp index c828bc21ee58..88a771bfeb60 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_admin.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_admin.cpp @@ -7,6 +7,90 @@ namespace NYdb { namespace NConsoleClient { +// FIXME: just reuse command one's +namespace { + TString FormatOption(const NLastGetopt::TOpt* option, const NColorizer::TColors& colors) { + using namespace NLastGetopt; + TStringStream result; + const TOpt::TShortNames& shorts = option->GetShortNames(); + const TOpt::TLongNames& longs = option->GetLongNames(); + + const size_t nopts = shorts.size() + longs.size(); + const bool multiple = 1 < nopts; + if (multiple) { + result << '{'; + } + for (size_t i = 0; i < nopts; ++i) { + if (multiple && 0 != i) { + result << '|'; + } + + if (i < shorts.size()) { // short + result << colors.GreenColor() << '-' << shorts[i] << colors.OldColor(); + } else { + result << colors.GreenColor() << "--" << longs[i - shorts.size()] << colors.OldColor(); + } + } + if (multiple) { + result << '}'; + } + + return result.Str(); + } + + // Option not to show in parent command help + bool NeedToHideOption(const NLastGetopt::TOpt* opt) { + if (opt->IsHidden()) { + return true; + } + for (const char shortName : opt->GetShortNames()) { + if (shortName == 'V' || shortName == 'h') + return true; + } + return false; + } + + void PrintOptionsDescription(IOutputStream& os, const NLastGetopt::TOpts* opts, NColorizer::TColors& colors, const TString& command) { + using namespace NLastGetopt; + NColorizer::TColors disabledColors(false); + os << " "; + bool firstPrintedOption = true; + for (size_t i = 0; i < opts->Opts_.size(); i++) { + const TOpt* opt = opts->Opts_[i].Get(); + if (NeedToHideOption(opt)) { + continue; + } + if (!firstPrintedOption) { + os << ", "; + } + os << FormatOption(opt, colors); + firstPrintedOption = false; + } + + os << Endl << " To get full description of these options run '" << command << "--help'."; + } + + void PrintParentOptions(TStringStream& stream, TClientCommand::TConfig& config, NColorizer::TColors& colors) { + bool foundRootParent = false; + TStringBuilder fullCommand; + for (const auto& parentCommand: config.ParentCommands) { + fullCommand << parentCommand.Name << " "; + if (parentCommand.Options) { + TString name = "Global"; + if (!foundRootParent) { + foundRootParent = true; + } else { + name = parentCommand.Name; + name[0] = toupper(name[0]); + stream << Endl << Endl; + } + stream << colors.BoldColor() << name << " options" << colors.OldColor() << ":" << Endl; + PrintOptionsDescription(stream, parentCommand.Options, colors, fullCommand); + } + } + } +} + class TCommandNode : public TClientCommandTree { public: TCommandNode() @@ -36,5 +120,29 @@ TCommandAdmin::TCommandAdmin() AddCommand(std::make_unique()); } +void TCommandAdmin::Config(TConfig& config) { + TClientCommand::Config(config); + SetFreeArgs(config); + TString commands; + SetFreeArgTitle(0, "", commands); + TStringStream stream; + NColorizer::TColors colors = NColorizer::AutoColors(Cout); + stream << Endl << Endl + << colors.BoldColor() + << "Commands in this subtree may damage your cluster if used wrong" << Endl + << "Due to dangerous nature of this commands ALL global parameters must be set explicitly" << Endl + << "Profiles are disabled by default, and used only if set explicitly (--profile )" << Endl + << "Some commands do not require global options which required otherwise" + << colors.OldColor(); + stream << Endl << Endl + << colors.BoldColor() << "Description" << colors.OldColor() << ": " << Description << Endl << Endl + << colors.BoldColor() << "Subcommands" << colors.OldColor() << ":" << Endl; + RenderCommandsDescription(stream, colors); + stream << Endl; + PrintParentOptions(stream, config, colors); + config.Opts->SetCmdLineDescr(stream.Str()); +} + + } } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_admin.h b/ydb/public/lib/ydb_cli/commands/ydb_admin.h index dbdc75d8c61b..d544544ec924 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_admin.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_admin.h @@ -8,6 +8,8 @@ namespace NConsoleClient { class TCommandAdmin : public TClientCommandTree { public: TCommandAdmin(); +protected: + virtual void Config(TConfig& config) override; }; } From 81f07bef485f3807e4e952060135ef1fce9e5fde Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Wed, 29 Jan 2025 21:20:10 +0000 Subject: [PATCH 09/54] WIP --- .../console/console__replace_yaml_config.cpp | 41 +++++++++---------- ydb/core/cms/console/console__scheme.h | 12 +++++- .../cms/console/console_configs_manager.cpp | 12 ++---- .../cms/console/console_configs_manager.h | 5 +-- ydb/core/cms/console/console_handshake.cpp | 4 +- 5 files changed, 37 insertions(+), 37 deletions(-) diff --git a/ydb/core/cms/console/console__replace_yaml_config.cpp b/ydb/core/cms/console/console__replace_yaml_config.cpp index 6373b86e3280..042fbc241cfe 100644 --- a/ydb/core/cms/console/console__replace_yaml_config.cpp +++ b/ydb/core/cms/console/console__replace_yaml_config.cpp @@ -66,24 +66,25 @@ class TConfigsManager::TTxReplaceYamlConfig : public TTransactionBaseValidateConfigAndReplaceMetadata(Config, Force, AllowUnknownFields); + TValidateConfigResult result = Self->ValidateConfigAndReplaceMetadata(Config, Force); + bool hasForbiddenUnknown = !result.UnknownFields.empty() && !AllowUnknownFields; if (result.ErrorReason) { HandleError(result.ErrorReason.value(), ctx); return true; } try { - auto fillResponse = [&](auto& ev, auto errorLevel){ + auto fillResponse = [&](auto& ev, auto errorLevel) { for (auto& [path, info] : result.UnknownFields) { auto *issue = ev->Record.AddIssues(); - issue->set_severity(errorLevel); - issue->set_message(TStringBuilder{} << "Unknown key# " << info.first << " in proto# " << info.second << " found in path# " << path); + issue->set_severity(errorLevel); + issue->set_message(TStringBuilder{} << "Unknown key# " << info.first << " in proto# " << info.second << " found in path# " << path); } for (auto& [path, info] : result.DeprecatedFields) { auto *issue = ev->Record.AddIssues(); - issue->set_severity(NYql::TSeverityIds::S_WARNING); - issue->set_message(TStringBuilder{} << "Deprecated key# " << info.first << " in proto# " << info.second << " found in path# " << path); + issue->set_severity(NYql::TSeverityIds::S_WARNING); + issue->set_message(TStringBuilder{} << "Deprecated key# " << info.first << " in proto# " << info.second << " found in path# " << path); } Response = MakeHolder(Sender, ctx.SelfID, ev.Release()); @@ -95,7 +96,7 @@ class TConfigsManager::TTxReplaceYamlConfig : public TTransactionBaseYamlConfig || Self->YamlDropped; if (!isMainConfig && !isDatabaseConfig) { Error = true; @@ -158,24 +159,22 @@ class TConfigsManager::TTxReplaceYamlConfig : public TTransactionBase().Key(Version + 1) - .Update(UpdatedConfig) - // set config dropped by default to support rollback to previous versions - // where new config layout is not supported - // it will lead to ignoring config from new versions - .Update(true); + db.Table().Key(Version + 1) + .Update(UpdatedConfig) + // set config dropped by default to support rollback to previous versions + // where new config layout is not supported + // it will lead to ignoring config from new versions + .Update(true); - /* Later we shift this boundary to support rollback and history */ - db.Table().Key(Version) - .Delete(); - } + /* Later we shift this boundary to support rollback and history */ + db.Table().Key(Version) + .Delete(); } - if (result.HasForbiddenUnknown) { + if (hasForbiddenUnknown) { Error = true; auto ev = MakeHolder(); ev->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); diff --git a/ydb/core/cms/console/console__scheme.h b/ydb/core/cms/console/console__scheme.h index b0e40979a994..8c7a0f355983 100644 --- a/ydb/core/cms/console/console__scheme.h +++ b/ydb/core/cms/console/console__scheme.h @@ -166,9 +166,19 @@ struct Schema : NIceDb::Schema { using TColumns = TableColumns; }; + struct PerTenantYamlConfig : Table<104> { + struct Path : Column<1, NScheme::NTypeIds::Utf8> {}; + struct Version : Column<2, NScheme::NTypeIds::Uint64> {}; + struct Config : Column<3, NScheme::NTypeIds::String> {}; + + + using TKey = TableKey; + using TColumns = TableColumns; + }; + using TTables = SchemaTables; + YamlConfig, PerTenantYamlConfig>; using TSettings = SchemaSettings, ExecutorLogFlushPeriod>; }; diff --git a/ydb/core/cms/console/console_configs_manager.cpp b/ydb/core/cms/console/console_configs_manager.cpp index 95890810aa9f..648be57f99e8 100644 --- a/ydb/core/cms/console/console_configs_manager.cpp +++ b/ydb/core/cms/console/console_configs_manager.cpp @@ -52,13 +52,11 @@ bool TConfigsManager::CheckConfig(const NKikimrConsole::TConfigsConfig &config, return true; } -TConfigsManager::TValidateConfigResult TConfigsManager::ValidateConfigAndReplaceMetadata(const TString &config, bool force, bool allowUnknownFields) { +TConfigsManager::TValidateConfigResult TConfigsManager::ValidateConfigAndReplaceMetadata(const TString &config, bool force) { TValidateConfigResult result; bool isDatabaseConfig = NYamlConfig::IsDatabaseConfig(config); if (isDatabaseConfig) { - result.HasForbiddenUnknown = false; - result.ValidationFinished = true; return result; } @@ -77,10 +75,7 @@ TConfigsManager::TValidateConfigResult TConfigsManager::ValidateConfigAndReplace .Cluster = result.Cluster, }); - result.HasForbiddenUnknown = false; if (result.UpdatedConfig != YamlConfig || YamlDropped) { - result.Modify = true; - auto tree = NFyaml::TDocument::Parse(result.UpdatedConfig); auto resolved = NYamlConfig::ResolveAll(tree); @@ -116,13 +111,11 @@ TConfigsManager::TValidateConfigResult TConfigsManager::ValidateConfigAndReplace result.UnknownFields[path] = info; } } - - result.HasForbiddenUnknown = !result.UnknownFields.empty() && !allowUnknownFields; - result.ValidationFinished = true; } } catch (const yexception &e) { result.ErrorReason = e.what(); } + return result; } @@ -691,6 +684,7 @@ void TConfigsManager::Handle(TEvConsole::TEvToggleConfigValidatorRequest::TPtr & void TConfigsManager::Handle(TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev, const TActorContext &ctx) { + // Validate? TxProcessor->ProcessTx(CreateTxReplaceYamlConfig(ev), ctx); } diff --git a/ydb/core/cms/console/console_configs_manager.h b/ydb/core/cms/console/console_configs_manager.h index fbff482a9a8d..babdc65113dd 100644 --- a/ydb/core/cms/console/console_configs_manager.h +++ b/ydb/core/cms/console/console_configs_manager.h @@ -34,14 +34,11 @@ class TConfigsManager : public TActorBootstrapped { struct TValidateConfigResult { std::optional ErrorReason; - bool Modify = false; TString UpdatedConfig; ui32 Version; TString Cluster; - bool HasForbiddenUnknown = false; TMap> DeprecatedFields; TMap> UnknownFields; - bool ValidationFinished = false; }; public: @@ -69,7 +66,7 @@ class TConfigsManager : public TActorBootstrapped { bool CheckConfig(const NKikimrConsole::TConfigsConfig &config, Ydb::StatusIds::StatusCode &code, TString &error); - TValidateConfigResult ValidateConfigAndReplaceMetadata(const TString& config, bool force = false, bool allowUnknownFields = false); + TValidateConfigResult ValidateConfigAndReplaceMetadata(const TString& config, bool force = false); void SendInReply(const TActorId& sender, const TActorId& icSession, std::unique_ptr ev, ui64 cookie = 0); diff --git a/ydb/core/cms/console/console_handshake.cpp b/ydb/core/cms/console/console_handshake.cpp index 43605b5ad391..78988680dfae 100644 --- a/ydb/core/cms/console/console_handshake.cpp +++ b/ydb/core/cms/console/console_handshake.cpp @@ -133,13 +133,13 @@ void TConfigsManager::Handle(TEvBlobStorage::TEvControllerValidateConfigRequest: return; } auto result = ValidateConfigAndReplaceMetadata(yamlConfig); - if (result.ErrorReason || result.HasForbiddenUnknown) { + if (result.ErrorReason || !result.UnknownFields.empty()) { record.SetStatus(NKikimrBlobStorage::TEvControllerValidateConfigResponse::ConfigNotValid); if (!result.ErrorReason) { record.SetErrorReason("has forbidden unknown fields"); } else { record.SetErrorReason(*result.ErrorReason); - if (result.HasForbiddenUnknown) { + if (!result.UnknownFields.empty()) { record.SetErrorReason(record.GetErrorReason() + " + has forbidden unknown fields"); } } From 8b42b6a61cde96604e76dc9b19dab570f99788b1 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Wed, 29 Jan 2025 22:15:30 +0000 Subject: [PATCH 10/54] WIP --- .../console/console__replace_yaml_config.cpp | 2 +- .../cms/console/console_configs_manager.cpp | 22 ++++++---- .../yaml_config/public/yaml_config.cpp | 41 +++++++++++++++++-- ydb/library/yaml_config/public/yaml_config.h | 33 +++++++++++---- 4 files changed, 79 insertions(+), 19 deletions(-) diff --git a/ydb/core/cms/console/console__replace_yaml_config.cpp b/ydb/core/cms/console/console__replace_yaml_config.cpp index 042fbc241cfe..d97f77bd7ee7 100644 --- a/ydb/core/cms/console/console__replace_yaml_config.cpp +++ b/ydb/core/cms/console/console__replace_yaml_config.cpp @@ -159,7 +159,7 @@ class TConfigsManager::TTxReplaceYamlConfig : public TTransactionBase().Key(Version + 1) diff --git a/ydb/core/cms/console/console_configs_manager.cpp b/ydb/core/cms/console/console_configs_manager.cpp index 648be57f99e8..297cc451d3ff 100644 --- a/ydb/core/cms/console/console_configs_manager.cpp +++ b/ydb/core/cms/console/console_configs_manager.cpp @@ -54,11 +54,6 @@ bool TConfigsManager::CheckConfig(const NKikimrConsole::TConfigsConfig &config, TConfigsManager::TValidateConfigResult TConfigsManager::ValidateConfigAndReplaceMetadata(const TString &config, bool force) { TValidateConfigResult result; - bool isDatabaseConfig = NYamlConfig::IsDatabaseConfig(config); - - if (isDatabaseConfig) { - return result; - } try { if (!force) { @@ -70,7 +65,7 @@ TConfigsManager::TValidateConfigResult TConfigsManager::ValidateConfigAndReplace result.Version = YamlVersion; } - result.UpdatedConfig = NYamlConfig::ReplaceMetadata(config, NYamlConfig::TMetadata{ + result.UpdatedConfig = NYamlConfig::ReplaceMetadata(config, NYamlConfig::TMainMetadata{ .Version = result.Version + 1, .Cluster = result.Cluster, }); @@ -684,8 +679,19 @@ void TConfigsManager::Handle(TEvConsole::TEvToggleConfigValidatorRequest::TPtr & void TConfigsManager::Handle(TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev, const TActorContext &ctx) { - // Validate? - TxProcessor->ProcessTx(CreateTxReplaceYamlConfig(ev), ctx); + auto metadata = NYamlConfig::GetGenericMetadata(ev->Get()->Record.GetRequest().config()); + + std::visit(TOverloaded{ + [&](const NYamlConfig::TMainMetadata& /* value */) { + TxProcessor->ProcessTx(CreateTxReplaceYamlConfig(ev), ctx); + }, + [](const NYamlConfig::TDatabaseMetadata& /* value */) { + // TODO + }, + [](auto&...) { + // ERROR + } + }, metadata); } void TConfigsManager::Handle(TEvConsole::TEvSetYamlConfigRequest::TPtr &ev, const TActorContext &ctx) diff --git a/ydb/library/yaml_config/public/yaml_config.cpp b/ydb/library/yaml_config/public/yaml_config.cpp index d7d5f06f3136..293e61d7f3cb 100644 --- a/ydb/library/yaml_config/public/yaml_config.cpp +++ b/ydb/library/yaml_config/public/yaml_config.cpp @@ -627,7 +627,7 @@ ui64 GetVersion(const TString& config) { return metadata.Version.value_or(0); } -TMetadata GetMetadata(const TString& config) { +TMainMetadata GetMetadata(const TString& config) { if (config.empty()) { return {}; } @@ -637,7 +637,7 @@ TMetadata GetMetadata(const TString& config) { if (auto node = doc.Root().Map()["metadata"]; node) { auto versionNode = node.Map()["version"]; auto clusterNode = node.Map()["cluster"]; - return TMetadata{ + return TMainMetadata{ .Version = versionNode ? std::optional{FromString(versionNode.Scalar())} : std::nullopt, .Cluster = clusterNode ? std::optional{clusterNode.Scalar()} : std::nullopt, }; @@ -646,6 +646,25 @@ TMetadata GetMetadata(const TString& config) { return {}; } +TDatabaseMetadata GetDatabaseMetadata(const TString& config) { + if (config.empty()) { + return {}; + } + + auto doc = NFyaml::TDocument::Parse(config); + + if (auto node = doc.Root().Map()["metadata"]; node) { + auto databaseNode = node.Map()["database"]; + auto versionNode = node.Map()["version"]; + return TDatabaseMetadata{ + .Version = versionNode ? std::optional{FromString(versionNode.Scalar())} : std::nullopt, + .Database = databaseNode ? std::optional{databaseNode.Scalar()} : std::nullopt, + }; + } + + return {}; +} + TVolatileMetadata GetVolatileMetadata(const TString& config) { if (config.empty()) { return {}; @@ -702,7 +721,7 @@ TString ReplaceMetadata(const TString& config, const std::function GetGenericMetadata(const TString& config) { + try { + auto doc = NFyaml::TDocument::Parse(config); + auto kind = doc.Root().Map().at("metadata").Map().at("kind").Scalar(); + if (kind == "MainConfig") { + return GetMetadata(config); + } else if (kind == "DatabaseConfig") { + return GetDatabaseMetadata(config); + } else { + return {}; + } + } catch (yexception& e) { + return {}; + } +} + } // namespace NKikimr::NYamlConfig template <> diff --git a/ydb/library/yaml_config/public/yaml_config.h b/ydb/library/yaml_config/public/yaml_config.h index af66579e3a69..21e9eed763fa 100644 --- a/ydb/library/yaml_config/public/yaml_config.h +++ b/ydb/library/yaml_config/public/yaml_config.h @@ -174,16 +174,11 @@ ui64 GetVersion(const TString& config); /** * Represents config metadata */ -struct TMetadata { +struct TMainMetadata { std::optional Version; std::optional Cluster; }; -/** - * Parses config metadata - */ -TMetadata GetMetadata(const TString& config); - /** * Represents volatile config metadata */ @@ -193,6 +188,30 @@ struct TVolatileMetadata { std::optional Id; }; +/** + * Represents database config metadata + */ +struct TDatabaseMetadata { + // maybe we should enforce Cluster as well + std::optional Version; + std::optional Database; +}; + +/** + * Parses config metadata + */ +std::variant GetGenericMetadata(const TString& config); + +/** + * Parses config metadata + */ +TMainMetadata GetMetadata(const TString& config); + +/** + * Parses database config metadata + */ +TDatabaseMetadata GetDatabaseMetadata(const TString& config); + /** * Parses volatile config metadata */ @@ -201,7 +220,7 @@ TVolatileMetadata GetVolatileMetadata(const TString& config); /** * Replaces metadata in config */ -TString ReplaceMetadata(const TString& config, const TMetadata& metadata); +TString ReplaceMetadata(const TString& config, const TMainMetadata& metadata); /** * Replaces volatile metadata in config From 1655b79040465b84767a2774b984a2c037969526 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Wed, 29 Jan 2025 23:16:58 +0000 Subject: [PATCH 11/54] WIP --- .../cms/console/console_configs_manager.cpp | 40 ++++++++++--------- .../cms/console/console_configs_manager.h | 9 +++-- ydb/core/cms/console/console_handshake.cpp | 14 ++++--- 3 files changed, 35 insertions(+), 28 deletions(-) diff --git a/ydb/core/cms/console/console_configs_manager.cpp b/ydb/core/cms/console/console_configs_manager.cpp index 297cc451d3ff..78c0af653cb6 100644 --- a/ydb/core/cms/console/console_configs_manager.cpp +++ b/ydb/core/cms/console/console_configs_manager.cpp @@ -52,33 +52,37 @@ bool TConfigsManager::CheckConfig(const NKikimrConsole::TConfigsConfig &config, return true; } -TConfigsManager::TValidateConfigResult TConfigsManager::ValidateConfigAndReplaceMetadata(const TString &config, bool force) { - TValidateConfigResult result; - +void TConfigsManager::ReplaceMainConfigMetadata(const TString &config, bool force, TUpdateConfigOpContext& opCtx) { try { if (!force) { auto metadata = NYamlConfig::GetMetadata(config); - result.Cluster = metadata.Cluster.value_or(TString("unknown")); - result.Version = metadata.Version.value_or(0); + opCtx.Cluster = metadata.Cluster.value_or(TString("unknown")); + opCtx.Version = metadata.Version.value_or(0); } else { - result.Cluster = ClusterName; - result.Version = YamlVersion; + opCtx.Cluster = ClusterName; + opCtx.Version = YamlVersion; } - result.UpdatedConfig = NYamlConfig::ReplaceMetadata(config, NYamlConfig::TMainMetadata{ - .Version = result.Version + 1, - .Cluster = result.Cluster, + opCtx.UpdatedConfig = NYamlConfig::ReplaceMetadata(config, NYamlConfig::TMainMetadata{ + .Version = opCtx.Version + 1, + .Cluster = opCtx.Cluster, }); + } catch (const yexception &e) { + opCtx.Error = e.what(); + } +} - if (result.UpdatedConfig != YamlConfig || YamlDropped) { - auto tree = NFyaml::TDocument::Parse(result.UpdatedConfig); +void TConfigsManager::ValidateMainConfig(TUpdateConfigOpContext& opCtx) { + try { + if (opCtx.UpdatedConfig != YamlConfig || YamlDropped) { + auto tree = NFyaml::TDocument::Parse(opCtx.UpdatedConfig); auto resolved = NYamlConfig::ResolveAll(tree); - if (ClusterName != result.Cluster) { + if (ClusterName != opCtx.Cluster) { ythrow yexception() << "ClusterName mismatch"; } - if (result.Version != YamlVersion) { + if (opCtx.Version != YamlVersion) { ythrow yexception() << "Version mismatch"; } @@ -101,17 +105,15 @@ TConfigsManager::TValidateConfigResult TConfigsManager::ValidateConfigAndReplace for (const auto& [path, info] : unknownFieldsCollector->GetUnknownKeys()) { if (deprecatedPaths.contains(path)) { - result.DeprecatedFields[path] = info; + opCtx.DeprecatedFields[path] = info; } else { - result.UnknownFields[path] = info; + opCtx.UnknownFields[path] = info; } } } } catch (const yexception &e) { - result.ErrorReason = e.what(); + opCtx.Error = e.what(); } - - return result; } void TConfigsManager::Bootstrap(const TActorContext &ctx) diff --git a/ydb/core/cms/console/console_configs_manager.h b/ydb/core/cms/console/console_configs_manager.h index babdc65113dd..9644be0530f2 100644 --- a/ydb/core/cms/console/console_configs_manager.h +++ b/ydb/core/cms/console/console_configs_manager.h @@ -32,8 +32,8 @@ class TConfigsManager : public TActorBootstrapped { using TBase = TActorBootstrapped; - struct TValidateConfigResult { - std::optional ErrorReason; + struct TUpdateConfigOpContext { + std::optional Error; TString UpdatedConfig; ui32 Version; TString Cluster; @@ -66,7 +66,10 @@ class TConfigsManager : public TActorBootstrapped { bool CheckConfig(const NKikimrConsole::TConfigsConfig &config, Ydb::StatusIds::StatusCode &code, TString &error); - TValidateConfigResult ValidateConfigAndReplaceMetadata(const TString& config, bool force = false); + + + void ReplaceMainConfigMetadata(const TString &config, bool force, TUpdateConfigOpContext& opCtx); + void ValidateMainConfig(TUpdateConfigOpContext& opCtx); void SendInReply(const TActorId& sender, const TActorId& icSession, std::unique_ptr ev, ui64 cookie = 0); diff --git a/ydb/core/cms/console/console_handshake.cpp b/ydb/core/cms/console/console_handshake.cpp index 78988680dfae..31dc8dc7ba6c 100644 --- a/ydb/core/cms/console/console_handshake.cpp +++ b/ydb/core/cms/console/console_handshake.cpp @@ -132,20 +132,22 @@ void TConfigsManager::Handle(TEvBlobStorage::TEvControllerValidateConfigRequest: if (!CheckSession(*ev, response, NKikimrBlobStorage::TEvControllerValidateConfigResponse::IdPipeServerMismatch)) { return; } - auto result = ValidateConfigAndReplaceMetadata(yamlConfig); - if (result.ErrorReason || !result.UnknownFields.empty()) { + TUpdateConfigOpContext opCtx; + ReplaceMainConfigMetadata(yamlConfig, false, opCtx); + ValidateMainConfig(opCtx); + if (opCtx.Error || !opCtx.UnknownFields.empty()) { record.SetStatus(NKikimrBlobStorage::TEvControllerValidateConfigResponse::ConfigNotValid); - if (!result.ErrorReason) { + if (!opCtx.Error) { record.SetErrorReason("has forbidden unknown fields"); } else { - record.SetErrorReason(*result.ErrorReason); - if (!result.UnknownFields.empty()) { + record.SetErrorReason(*opCtx.Error); + if (!opCtx.UnknownFields.empty()) { record.SetErrorReason(record.GetErrorReason() + " + has forbidden unknown fields"); } } } else { record.SetStatus(NKikimrBlobStorage::TEvControllerValidateConfigResponse::ConfigIsValid); - record.SetYAML(result.UpdatedConfig); + record.SetYAML(opCtx.UpdatedConfig); } SendInReply(ev->Sender, ev->InterconnectSession, std::move(response), ev->Cookie); } From 940a87390a7c7680665b2074b1e7c0f3c7172159 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Wed, 29 Jan 2025 23:34:56 +0000 Subject: [PATCH 12/54] WIP --- .../console/console__replace_yaml_config.cpp | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/ydb/core/cms/console/console__replace_yaml_config.cpp b/ydb/core/cms/console/console__replace_yaml_config.cpp index d97f77bd7ee7..9e936d44716f 100644 --- a/ydb/core/cms/console/console__replace_yaml_config.cpp +++ b/ydb/core/cms/console/console__replace_yaml_config.cpp @@ -66,22 +66,26 @@ class TConfigsManager::TTxReplaceYamlConfig : public TTransactionBaseValidateConfigAndReplaceMetadata(Config, Force); - bool hasForbiddenUnknown = !result.UnknownFields.empty() && !AllowUnknownFields; - if (result.ErrorReason) { - HandleError(result.ErrorReason.value(), ctx); + + TUpdateConfigOpContext opCtx; + Self->ReplaceMainConfigMetadata(Config, false, opCtx); + Self->ValidateMainConfig(opCtx); + + bool hasForbiddenUnknown = !opCtx.UnknownFields.empty() && !AllowUnknownFields; + if (opCtx.Error) { + HandleError(opCtx.Error.value(), ctx); return true; } try { auto fillResponse = [&](auto& ev, auto errorLevel) { - for (auto& [path, info] : result.UnknownFields) { + for (auto& [path, info] : opCtx.UnknownFields) { auto *issue = ev->Record.AddIssues(); issue->set_severity(errorLevel); issue->set_message(TStringBuilder{} << "Unknown key# " << info.first << " in proto# " << info.second << " found in path# " << path); } - for (auto& [path, info] : result.DeprecatedFields) { + for (auto& [path, info] : opCtx.DeprecatedFields) { auto *issue = ev->Record.AddIssues(); issue->set_severity(NYql::TSeverityIds::S_WARNING); issue->set_message(TStringBuilder{} << "Deprecated key# " << info.first << " in proto# " << info.second << " found in path# " << path); @@ -93,10 +97,10 @@ class TConfigsManager::TTxReplaceYamlConfig : public TTransactionBaseYamlConfig || Self->YamlDropped; + Version = opCtx.Version; + UpdatedConfig = opCtx.UpdatedConfig; + Cluster = opCtx.Cluster; + Modify = opCtx.UpdatedConfig != Self->YamlConfig || Self->YamlDropped; if (!isMainConfig && !isDatabaseConfig) { Error = true; From 396c8b80e540a62e65c2630e4d2a85c2c5d578c2 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Thu, 30 Jan 2025 11:47:57 +0000 Subject: [PATCH 13/54] WIP --- .../console/console__replace_yaml_config.cpp | 101 +++++++++++++----- .../cms/console/console_configs_manager.cpp | 44 ++++++-- .../cms/console/console_configs_manager.h | 10 +- 3 files changed, 118 insertions(+), 37 deletions(-) diff --git a/ydb/core/cms/console/console__replace_yaml_config.cpp b/ydb/core/cms/console/console__replace_yaml_config.cpp index 9e936d44716f..d792dd05ff7e 100644 --- a/ydb/core/cms/console/console__replace_yaml_config.cpp +++ b/ydb/core/cms/console/console__replace_yaml_config.cpp @@ -13,33 +13,38 @@ namespace NKikimr::NConsole { using namespace NKikimrConsole; -class TConfigsManager::TTxReplaceYamlConfig : public TTransactionBase { +class TConfigsManager::TTxReplaceYamlConfigBase + : public TTransactionBase +{ template - TTxReplaceYamlConfig(TConfigsManager *self, - T &ev, - bool force) - : TBase(self) - , Config(ev->Get()->Record.GetRequest().config()) - , Peer(ev->Get()->Record.GetPeerName()) - , Sender(ev->Sender) - , UserToken(ev->Get()->Record.GetUserToken()) - , Force(force) - , AllowUnknownFields(ev->Get()->Record.GetRequest().allow_unknown_fields()) - , DryRun(ev->Get()->Record.GetRequest().dry_run()) - , Database(ev->Get()->Record.HasDatabase() ? TMaybe{ev->Get()->Record.GetDatabase()} : TMaybe{}) + TTxReplaceYamlConfigBase( + TConfigsManager *self, + T &ev, + bool force) + : TBase(self) + , Config(ev->Get()->Record.GetRequest().config()) + , Peer(ev->Get()->Record.GetPeerName()) + , Sender(ev->Sender) + , UserToken(ev->Get()->Record.GetUserToken()) + , Force(force) + , AllowUnknownFields(ev->Get()->Record.GetRequest().allow_unknown_fields()) + , DryRun(ev->Get()->Record.GetRequest().dry_run()) + , Database(ev->Get()->Record.HasDatabase() ? TMaybe{ev->Get()->Record.GetDatabase()} : TMaybe{}) { } public: - TTxReplaceYamlConfig(TConfigsManager *self, - TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev) - : TTxReplaceYamlConfig(self, ev, false) + TTxReplaceYamlConfigBase( + TConfigsManager *self, + TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev) + : TTxReplaceYamlConfigBase(self, ev, false) { } - TTxReplaceYamlConfig(TConfigsManager *self, - TEvConsole::TEvSetYamlConfigRequest::TPtr &ev) - : TTxReplaceYamlConfig(self, ev, true) + TTxReplaceYamlConfigBase( + TConfigsManager *self, + TEvConsole::TEvSetYamlConfigRequest::TPtr &ev) + : TTxReplaceYamlConfigBase(self, ev, true) { } @@ -247,7 +252,7 @@ class TConfigsManager::TTxReplaceYamlConfig : public TTransactionBase(Sender, ctx.SelfID, ev.Release()); } -private: +protected: const TString Config; const TString Peer; const TActorId Sender; @@ -267,14 +272,62 @@ class TConfigsManager::TTxReplaceYamlConfig : public TTransactionBaseGet()->Record.GetRequest().config()); std::visit(TOverloaded{ - [&](const NYamlConfig::TMainMetadata& /* value */) { - TxProcessor->ProcessTx(CreateTxReplaceYamlConfig(ev), ctx); - }, - [](const NYamlConfig::TDatabaseMetadata& /* value */) { - // TODO - }, - [](auto&...) { - // ERROR - } + [&](const NYamlConfig::TMainMetadata& /* value */) { + TxProcessor->ProcessTx(CreateTxReplaceMainYamlConfig(ev), ctx); + }, + [&](const NYamlConfig::TDatabaseMetadata& /* value */) { + TxProcessor->ProcessTx(CreateTxReplaceDatabaseYamlConfig(ev), ctx); + }, + [&](auto&...) { + auto resp = MakeHolder(); + resp->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); + auto *issue = resp->Record.AddIssues(); + issue->set_severity(NYql::TSeverityIds::S_ERROR); + issue->set_message("Unknown config kind or format"); + auto response = MakeHolder(ev->Sender, ctx.SelfID, resp.Release()); + ctx.Send(response.Release()); + } }, metadata); } void TConfigsManager::Handle(TEvConsole::TEvSetYamlConfigRequest::TPtr &ev, const TActorContext &ctx) { - TxProcessor->ProcessTx(CreateTxSetYamlConfig(ev), ctx); + auto metadata = NYamlConfig::GetGenericMetadata(ev->Get()->Record.GetRequest().config()); + + std::visit(TOverloaded{ + [&](const NYamlConfig::TMainMetadata& /* value */) { + TxProcessor->ProcessTx(CreateTxSetMainYamlConfig(ev), ctx); + }, + [&](const NYamlConfig::TDatabaseMetadata& /* value */) { + TxProcessor->ProcessTx(CreateTxSetDatabaseYamlConfig(ev), ctx); + }, + [&](auto&...) { + auto resp = MakeHolder(); + resp->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); + auto *issue = resp->Record.AddIssues(); + issue->set_severity(NYql::TSeverityIds::S_ERROR); + issue->set_message("Unknown config kind or format"); + auto response = MakeHolder(ev->Sender, ctx.SelfID, resp.Release()); + ctx.Send(response.Release()); + } + }, metadata); } void TConfigsManager::Handle(TEvConsole::TEvDropConfigRequest::TPtr &ev, const TActorContext &ctx) diff --git a/ydb/core/cms/console/console_configs_manager.h b/ydb/core/cms/console/console_configs_manager.h index 9644be0530f2..8971a1f2a78c 100644 --- a/ydb/core/cms/console/console_configs_manager.h +++ b/ydb/core/cms/console/console_configs_manager.h @@ -128,7 +128,9 @@ class TConfigsManager : public TActorBootstrapped { class TTxUpdateLastProvidedConfig; class TTxGetLogTail; class TTxLogCleanup; - class TTxReplaceYamlConfig; + class TTxReplaceYamlConfigBase; + class TTxReplaceMainYamlConfig; + class TTxReplaceDatabaseYamlConfig; class TTxDropYamlConfig; class TTxGetYamlConfig; class TTxGetYamlMetadata; @@ -145,8 +147,10 @@ class TConfigsManager : public TActorBootstrapped { ITransaction *CreateTxUpdateLastProvidedConfig(TEvConsole::TEvConfigNotificationResponse::TPtr &ev); ITransaction *CreateTxGetLogTail(TEvConsole::TEvGetLogTailRequest::TPtr &ev); ITransaction *CreateTxLogCleanup(); - ITransaction *CreateTxReplaceYamlConfig(TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev); - ITransaction *CreateTxSetYamlConfig(TEvConsole::TEvSetYamlConfigRequest::TPtr &ev); + ITransaction *CreateTxReplaceMainYamlConfig(TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev); + ITransaction *CreateTxReplaceDatabaseYamlConfig(TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev); + ITransaction *CreateTxSetMainYamlConfig(TEvConsole::TEvSetYamlConfigRequest::TPtr &ev); + ITransaction *CreateTxSetDatabaseYamlConfig(TEvConsole::TEvSetYamlConfigRequest::TPtr &ev); ITransaction *CreateTxDropYamlConfig(TEvConsole::TEvDropConfigRequest::TPtr &ev); ITransaction *CreateTxGetYamlConfig(TEvConsole::TEvGetAllConfigsRequest::TPtr &ev); ITransaction *CreateTxGetYamlMetadata(TEvConsole::TEvGetAllMetadataRequest::TPtr &ev); From dcd2644a92aa95ffdf6b00d2b4e1120186719747 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Thu, 30 Jan 2025 12:02:55 +0000 Subject: [PATCH 14/54] WIP --- .../console/console__replace_yaml_config.cpp | 314 ++++++++++-------- 1 file changed, 179 insertions(+), 135 deletions(-) diff --git a/ydb/core/cms/console/console__replace_yaml_config.cpp b/ydb/core/cms/console/console__replace_yaml_config.cpp index d792dd05ff7e..dc5381973ca3 100644 --- a/ydb/core/cms/console/console__replace_yaml_config.cpp +++ b/ydb/core/cms/console/console__replace_yaml_config.cpp @@ -68,141 +68,6 @@ class TConfigsManager::TTxReplaceYamlConfigBase Self->Logger.DbLogData(UserToken.GetUserSID(), logData, txc, ctx); } - bool Execute(TTransactionContext &txc, const TActorContext &ctx) override - { - NIceDb::TNiceDb db(txc.DB); - - TUpdateConfigOpContext opCtx; - Self->ReplaceMainConfigMetadata(Config, false, opCtx); - Self->ValidateMainConfig(opCtx); - - bool hasForbiddenUnknown = !opCtx.UnknownFields.empty() && !AllowUnknownFields; - if (opCtx.Error) { - HandleError(opCtx.Error.value(), ctx); - return true; - } - - try { - auto fillResponse = [&](auto& ev, auto errorLevel) { - for (auto& [path, info] : opCtx.UnknownFields) { - auto *issue = ev->Record.AddIssues(); - issue->set_severity(errorLevel); - issue->set_message(TStringBuilder{} << "Unknown key# " << info.first << " in proto# " << info.second << " found in path# " << path); - } - - for (auto& [path, info] : opCtx.DeprecatedFields) { - auto *issue = ev->Record.AddIssues(); - issue->set_severity(NYql::TSeverityIds::S_WARNING); - issue->set_message(TStringBuilder{} << "Deprecated key# " << info.first << " in proto# " << info.second << " found in path# " << path); - } - - Response = MakeHolder(Sender, ctx.SelfID, ev.Release()); - }; - - bool isMainConfig = NYamlConfig::IsMainConfig(Config); - bool isDatabaseConfig = NYamlConfig::IsDatabaseConfig(Config); - - Version = opCtx.Version; - UpdatedConfig = opCtx.UpdatedConfig; - Cluster = opCtx.Cluster; - Modify = opCtx.UpdatedConfig != Self->YamlConfig || Self->YamlDropped; - - if (!isMainConfig && !isDatabaseConfig) { - Error = true; - auto ev = MakeHolder(); - - auto *issue = ev->Record.AddIssues(); - ErrorReason = "Unknown config kind"; - issue->set_severity(NYql::TSeverityIds::S_ERROR); - issue->set_message(ErrorReason); - ev->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); - Response = MakeHolder(Sender, ctx.SelfID, ev.Release()); - return true; - } - - if (Database && isMainConfig) { - WarnDatabaseByPass = true; - } - - if (Database && isDatabaseConfig && !AppData(ctx)->FeatureFlags.GetPerDatabaseConfigAllowed()) { - Error = true; - auto ev = MakeHolder(); - - auto *issue = ev->Record.AddIssues(); - ErrorReason = "Per database config is disabled"; - issue->set_severity(NYql::TSeverityIds::S_ERROR); - issue->set_message(ErrorReason); - ev->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); - Response = MakeHolder(Sender, ctx.SelfID, ev.Release()); - return true; - } - - if (Database && isDatabaseConfig) { - Self->YamlConfigPerDatabase[*Database] = Config; - // FIXME - Modify = true; - UpdatedConfig = Self->YamlConfig; - // TODO - // 1) send - // 1) persist - // 2) validate - // 3) support force - // 4) support dry-run - // 5) support audit - auto ev = MakeHolder(); - fillResponse(ev, NYql::TSeverityIds::S_WARNING); - - return true; - } - - if (!isMainConfig) { - Error = true; - auto ev = MakeHolder(); - - auto *issue = ev->Record.AddIssues(); - ErrorReason = "Invalid config kind"; - issue->set_severity(NYql::TSeverityIds::S_ERROR); - issue->set_message(ErrorReason); - ev->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); - Response = MakeHolder(Sender, ctx.SelfID, ev.Release()); - return true; - } - - if (!DryRun && hasForbiddenUnknown) { - DoInternalAudit(txc, ctx); - - db.Table().Key(Version + 1) - .Update(UpdatedConfig) - // set config dropped by default to support rollback to previous versions - // where new config layout is not supported - // it will lead to ignoring config from new versions - .Update(true); - - /* Later we shift this boundary to support rollback and history */ - db.Table().Key(Version) - .Delete(); - } - - if (hasForbiddenUnknown) { - Error = true; - auto ev = MakeHolder(); - ev->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); - ErrorReason = "Unknown keys in config."; - fillResponse(ev, NYql::TSeverityIds::S_ERROR); - } else if (!Force) { - auto ev = MakeHolder(); - fillResponse(ev, NYql::TSeverityIds::S_WARNING); - } else { - auto ev = MakeHolder(); - fillResponse(ev, NYql::TSeverityIds::S_WARNING); - } - } - catch (const yexception& ex) { - HandleError(ex.what(), ctx); - } - return true; - } - void Complete(const TActorContext &ctx) override { LOG_DEBUG(ctx, NKikimrServices::CMS_CONFIGS, "TTxReplaceYamlConfig Complete"); @@ -289,6 +154,82 @@ class TConfigsManager::TTxReplaceMainYamlConfig : TTxReplaceYamlConfigBase(self, ev) { } + + bool Execute(TTransactionContext &txc, const TActorContext &ctx) override + { + NIceDb::TNiceDb db(txc.DB); + + TUpdateConfigOpContext opCtx; + Self->ReplaceMainConfigMetadata(Config, false, opCtx); + Self->ValidateMainConfig(opCtx); + + bool hasForbiddenUnknown = !opCtx.UnknownFields.empty() && !AllowUnknownFields; + if (opCtx.Error) { + HandleError(opCtx.Error.value(), ctx); + return true; + } + + try { + auto fillResponse = [&](auto& ev, auto errorLevel) { + for (auto& [path, info] : opCtx.UnknownFields) { + auto *issue = ev->Record.AddIssues(); + issue->set_severity(errorLevel); + issue->set_message(TStringBuilder{} << "Unknown key# " << info.first << " in proto# " << info.second << " found in path# " << path); + } + + for (auto& [path, info] : opCtx.DeprecatedFields) { + auto *issue = ev->Record.AddIssues(); + issue->set_severity(NYql::TSeverityIds::S_WARNING); + issue->set_message(TStringBuilder{} << "Deprecated key# " << info.first << " in proto# " << info.second << " found in path# " << path); + } + + Response = MakeHolder(Sender, ctx.SelfID, ev.Release()); + }; + + Version = opCtx.Version; + UpdatedConfig = opCtx.UpdatedConfig; + Cluster = opCtx.Cluster; + Modify = opCtx.UpdatedConfig != Self->YamlConfig || Self->YamlDropped; + + if (Database) { + WarnDatabaseByPass = true; + } + + if (!DryRun && !hasForbiddenUnknown) { + DoInternalAudit(txc, ctx); + + db.Table().Key(Version + 1) + .Update(UpdatedConfig) + // set config dropped by default to support rollback to previous versions + // where new config layout is not supported + // it will lead to ignoring config from new versions + .Update(true); + + /* Later we shift this boundary to support rollback and history */ + db.Table().Key(Version) + .Delete(); + } + + if (hasForbiddenUnknown) { + Error = true; + auto ev = MakeHolder(); + ev->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); + ErrorReason = "Unknown keys in config."; + fillResponse(ev, NYql::TSeverityIds::S_ERROR); + } else if (!Force) { + auto ev = MakeHolder(); + fillResponse(ev, NYql::TSeverityIds::S_WARNING); + } else { + auto ev = MakeHolder(); + fillResponse(ev, NYql::TSeverityIds::S_WARNING); + } + } + catch (const yexception& ex) { + HandleError(ex.what(), ctx); + } + + return true; + } }; class TConfigsManager::TTxReplaceDatabaseYamlConfig @@ -308,6 +249,109 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig : TTxReplaceYamlConfigBase(self, ev) { } + + bool Execute(TTransactionContext &txc, const TActorContext &ctx) override + { + NIceDb::TNiceDb db(txc.DB); + + TUpdateConfigOpContext opCtx; + Self->ReplaceMainConfigMetadata(Config, false, opCtx); + Self->ValidateMainConfig(opCtx); + + bool hasForbiddenUnknown = !opCtx.UnknownFields.empty() && !AllowUnknownFields; + if (opCtx.Error) { + HandleError(opCtx.Error.value(), ctx); + return true; + } + + try { + auto fillResponse = [&](auto& ev, auto errorLevel) { + for (auto& [path, info] : opCtx.UnknownFields) { + auto *issue = ev->Record.AddIssues(); + issue->set_severity(errorLevel); + issue->set_message(TStringBuilder{} << "Unknown key# " << info.first << " in proto# " << info.second << " found in path# " << path); + } + + for (auto& [path, info] : opCtx.DeprecatedFields) { + auto *issue = ev->Record.AddIssues(); + issue->set_severity(NYql::TSeverityIds::S_WARNING); + issue->set_message(TStringBuilder{} << "Deprecated key# " << info.first << " in proto# " << info.second << " found in path# " << path); + } + + Response = MakeHolder(Sender, ctx.SelfID, ev.Release()); + }; + + Version = opCtx.Version; + UpdatedConfig = opCtx.UpdatedConfig; + Cluster = opCtx.Cluster; + Modify = opCtx.UpdatedConfig != Self->YamlConfig || Self->YamlDropped; + + // if (!Database) { FIXME: extract database from config itself + // WarnDatabaseByPass = true; + // } + + if (!AppData(ctx)->FeatureFlags.GetPerDatabaseConfigAllowed()) { + Error = true; + auto ev = MakeHolder(); + + auto *issue = ev->Record.AddIssues(); + ErrorReason = "Per database config is disabled"; + issue->set_severity(NYql::TSeverityIds::S_ERROR); + issue->set_message(ErrorReason); + ev->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); + Response = MakeHolder(Sender, ctx.SelfID, ev.Release()); + return true; + } + + Self->YamlConfigPerDatabase[*Database] = Config; + // FIXME + Modify = true; + UpdatedConfig = Self->YamlConfig; + // TODO + // 1) send + // 1) persist + // 2) validate + // 3) support force + // 4) support dry-run + // 5) support audit + auto ev = MakeHolder(); + fillResponse(ev, NYql::TSeverityIds::S_WARNING); + + if (!DryRun && !hasForbiddenUnknown) { + DoInternalAudit(txc, ctx); + + db.Table().Key(Version + 1) + .Update(UpdatedConfig) + // set config dropped by default to support rollback to previous versions + // where new config layout is not supported + // it will lead to ignoring config from new versions + .Update(true); + + /* Later we shift this boundary to support rollback and history */ + db.Table().Key(Version) + .Delete(); + } + + if (hasForbiddenUnknown) { + Error = true; + auto ev = MakeHolder(); + ev->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); + ErrorReason = "Unknown keys in config."; + fillResponse(ev, NYql::TSeverityIds::S_ERROR); + } else if (!Force) { + auto ev = MakeHolder(); + fillResponse(ev, NYql::TSeverityIds::S_WARNING); + } else { + auto ev = MakeHolder(); + fillResponse(ev, NYql::TSeverityIds::S_WARNING); + } + } + catch (const yexception& ex) { + HandleError(ex.what(), ctx); + } + + return true; + } }; ITransaction *TConfigsManager::CreateTxReplaceMainYamlConfig(TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev) From 9afe5d9743eb34fefdc72e6437b5a9e7154cfeb9 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Thu, 30 Jan 2025 12:11:15 +0000 Subject: [PATCH 15/54] WIP --- .../console/console__replace_yaml_config.cpp | 119 ++++++++++-------- 1 file changed, 64 insertions(+), 55 deletions(-) diff --git a/ydb/core/cms/console/console__replace_yaml_config.cpp b/ydb/core/cms/console/console__replace_yaml_config.cpp index dc5381973ca3..3d7964d4df34 100644 --- a/ydb/core/cms/console/console__replace_yaml_config.cpp +++ b/ydb/core/cms/console/console__replace_yaml_config.cpp @@ -58,7 +58,7 @@ class TConfigsManager::TTxReplaceYamlConfigBase auto& yamlConfigChange = *logData.MutableYamlConfigChange(); yamlConfigChange.SetOldYamlConfig(Self->YamlConfig); - yamlConfigChange.SetNewYamlConfig(UpdatedConfig); + // yamlConfigChange.SetNewYamlConfig(UpdatedConfig); // FIXME for (auto& [id, config] : Self->VolatileYamlConfigs) { auto& oldVolatileConfig = *yamlConfigChange.AddOldVolatileYamlConfigs(); oldVolatileConfig.SetId(id); @@ -68,44 +68,6 @@ class TConfigsManager::TTxReplaceYamlConfigBase Self->Logger.DbLogData(UserToken.GetUserSID(), logData, txc, ctx); } - void Complete(const TActorContext &ctx) override - { - LOG_DEBUG(ctx, NKikimrServices::CMS_CONFIGS, "TTxReplaceYamlConfig Complete"); - - ctx.Send(Response.Release()); - - if (!Error && Modify && !DryRun) { - AuditLogReplaceConfigTransaction( - /* peer = */ Peer, - /* userSID = */ UserToken.GetUserSID(), - /* sanitizedToken = */ UserToken.GetSanitizedToken(), - /* oldConfig = */ Self->YamlConfig, - /* newConfig = */ Config, - /* reason = */ {}, - /* success = */ true); - - Self->YamlVersion = Version + 1; - Self->YamlConfig = UpdatedConfig; - Self->YamlDropped = false; - - Self->VolatileYamlConfigs.clear(); - - auto resp = MakeHolder(Self->YamlConfig, Self->YamlConfigPerDatabase); - ctx.Send(Self->ConfigsProvider, resp.Release()); - } else if (Error && !DryRun) { - AuditLogReplaceConfigTransaction( - /* peer = */ Peer, - /* userSID = */ UserToken.GetUserSID(), - /* sanitizedToken = */ UserToken.GetSanitizedToken(), - /* oldConfig = */ Self->YamlConfig, - /* newConfig = */ Config, - /* reason = */ ErrorReason, - /* success = */ false); - } - - Self->TxProcessor->TxCompleted(this, ctx); - } - void HandleError(const TString& error, const TActorContext& ctx) { Error = true; auto ev = MakeHolder(); @@ -132,7 +94,6 @@ class TConfigsManager::TTxReplaceYamlConfigBase TSimpleSharedPtr UnknownFieldsCollector = nullptr; ui32 Version; TString Cluster; - TString UpdatedConfig; TMaybe Database; bool WarnDatabaseByPass = false; }; @@ -230,6 +191,47 @@ class TConfigsManager::TTxReplaceMainYamlConfig return true; } + + void Complete(const TActorContext &ctx) override + { + LOG_DEBUG(ctx, NKikimrServices::CMS_CONFIGS, "TTxReplaceYamlConfig Complete"); + + ctx.Send(Response.Release()); + + if (!Error && Modify && !DryRun) { + AuditLogReplaceConfigTransaction( + /* peer = */ Peer, + /* userSID = */ UserToken.GetUserSID(), + /* sanitizedToken = */ UserToken.GetSanitizedToken(), + /* oldConfig = */ Self->YamlConfig, + /* newConfig = */ Config, + /* reason = */ {}, + /* success = */ true); + + Self->YamlVersion = Version + 1; + Self->YamlConfig = UpdatedConfig; + Self->YamlDropped = false; + + Self->VolatileYamlConfigs.clear(); + + auto resp = MakeHolder(Self->YamlConfig, Self->YamlConfigPerDatabase); + ctx.Send(Self->ConfigsProvider, resp.Release()); + } else if (Error && !DryRun) { + AuditLogReplaceConfigTransaction( + /* peer = */ Peer, + /* userSID = */ UserToken.GetUserSID(), + /* sanitizedToken = */ UserToken.GetSanitizedToken(), + /* oldConfig = */ Self->YamlConfig, + /* newConfig = */ Config, + /* reason = */ ErrorReason, + /* success = */ false); + } + + Self->TxProcessor->TxCompleted(this, ctx); + } + +private: + TString UpdatedConfig; }; class TConfigsManager::TTxReplaceDatabaseYamlConfig @@ -282,9 +284,9 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig }; Version = opCtx.Version; - UpdatedConfig = opCtx.UpdatedConfig; + // UpdatedConfig = opCtx.UpdatedConfig; Cluster = opCtx.Cluster; - Modify = opCtx.UpdatedConfig != Self->YamlConfig || Self->YamlDropped; + // Modify = opCtx.UpdatedConfig != Self->YamlConfig || Self->YamlDropped; // if (!Database) { FIXME: extract database from config itself // WarnDatabaseByPass = true; @@ -306,7 +308,6 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig Self->YamlConfigPerDatabase[*Database] = Config; // FIXME Modify = true; - UpdatedConfig = Self->YamlConfig; // TODO // 1) send // 1) persist @@ -317,20 +318,20 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig auto ev = MakeHolder(); fillResponse(ev, NYql::TSeverityIds::S_WARNING); - if (!DryRun && !hasForbiddenUnknown) { - DoInternalAudit(txc, ctx); + // if (!DryRun && !hasForbiddenUnknown) { + // DoInternalAudit(txc, ctx); - db.Table().Key(Version + 1) - .Update(UpdatedConfig) - // set config dropped by default to support rollback to previous versions - // where new config layout is not supported - // it will lead to ignoring config from new versions - .Update(true); + // db.Table().Key(Version + 1) + // .Update(UpdatedConfig) + // // set config dropped by default to support rollback to previous versions + // // where new config layout is not supported + // // it will lead to ignoring config from new versions + // .Update(true); - /* Later we shift this boundary to support rollback and history */ - db.Table().Key(Version) - .Delete(); - } + // /* Later we shift this boundary to support rollback and history */ + // db.Table().Key(Version) + // .Delete(); + // } if (hasForbiddenUnknown) { Error = true; @@ -352,6 +353,14 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig return true; } + + void Complete(const TActorContext &ctx) override + { + Y_UNUSED(ctx); // TODO + } + +private: + // TODO }; ITransaction *TConfigsManager::CreateTxReplaceMainYamlConfig(TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev) From d330e717147f1511810336acf8a678465b21e681 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Thu, 30 Jan 2025 12:22:29 +0000 Subject: [PATCH 16/54] WIP --- .../console/console__replace_yaml_config.cpp | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/ydb/core/cms/console/console__replace_yaml_config.cpp b/ydb/core/cms/console/console__replace_yaml_config.cpp index 3d7964d4df34..a6a2bd4dfa8a 100644 --- a/ydb/core/cms/console/console__replace_yaml_config.cpp +++ b/ydb/core/cms/console/console__replace_yaml_config.cpp @@ -48,6 +48,22 @@ class TConfigsManager::TTxReplaceYamlConfigBase { } + THolder FillResponse(const TUpdateConfigOpContext& opCtx, auto& ev, auto errorLevel, const TActorContext &ctx) { + for (auto& [path, info] : opCtx.UnknownFields) { + auto *issue = ev->Record.AddIssues(); + issue->set_severity(errorLevel); + issue->set_message(TStringBuilder{} << "Unknown key# " << info.first << " in proto# " << info.second << " found in path# " << path); + } + + for (auto& [path, info] : opCtx.DeprecatedFields) { + auto *issue = ev->Record.AddIssues(); + issue->set_severity(NYql::TSeverityIds::S_WARNING); + issue->set_message(TStringBuilder{} << "Deprecated key# " << info.first << " in proto# " << info.second << " found in path# " << path); + } + + return MakeHolder(Sender, ctx.SelfID, ev.Release()); + } + void DoInternalAudit(TTransactionContext &txc, const TActorContext &ctx) { auto logData = NKikimrConsole::TLogRecordData{}; @@ -131,22 +147,6 @@ class TConfigsManager::TTxReplaceMainYamlConfig } try { - auto fillResponse = [&](auto& ev, auto errorLevel) { - for (auto& [path, info] : opCtx.UnknownFields) { - auto *issue = ev->Record.AddIssues(); - issue->set_severity(errorLevel); - issue->set_message(TStringBuilder{} << "Unknown key# " << info.first << " in proto# " << info.second << " found in path# " << path); - } - - for (auto& [path, info] : opCtx.DeprecatedFields) { - auto *issue = ev->Record.AddIssues(); - issue->set_severity(NYql::TSeverityIds::S_WARNING); - issue->set_message(TStringBuilder{} << "Deprecated key# " << info.first << " in proto# " << info.second << " found in path# " << path); - } - - Response = MakeHolder(Sender, ctx.SelfID, ev.Release()); - }; - Version = opCtx.Version; UpdatedConfig = opCtx.UpdatedConfig; Cluster = opCtx.Cluster; @@ -176,13 +176,13 @@ class TConfigsManager::TTxReplaceMainYamlConfig auto ev = MakeHolder(); ev->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); ErrorReason = "Unknown keys in config."; - fillResponse(ev, NYql::TSeverityIds::S_ERROR); + Response = FillResponse(opCtx, ev, NYql::TSeverityIds::S_ERROR, ctx); } else if (!Force) { auto ev = MakeHolder(); - fillResponse(ev, NYql::TSeverityIds::S_WARNING); + Response = FillResponse(opCtx, ev, NYql::TSeverityIds::S_WARNING, ctx); } else { auto ev = MakeHolder(); - fillResponse(ev, NYql::TSeverityIds::S_WARNING); + Response = FillResponse(opCtx, ev, NYql::TSeverityIds::S_WARNING, ctx); } } catch (const yexception& ex) { From ab860646395f4cbdcdf0c76038af49e6b6e4f44b Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Thu, 30 Jan 2025 12:28:14 +0000 Subject: [PATCH 17/54] WIP --- .../console/console__replace_yaml_config.cpp | 25 ++++--------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/ydb/core/cms/console/console__replace_yaml_config.cpp b/ydb/core/cms/console/console__replace_yaml_config.cpp index a6a2bd4dfa8a..1c7f6228d458 100644 --- a/ydb/core/cms/console/console__replace_yaml_config.cpp +++ b/ydb/core/cms/console/console__replace_yaml_config.cpp @@ -267,22 +267,6 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig } try { - auto fillResponse = [&](auto& ev, auto errorLevel) { - for (auto& [path, info] : opCtx.UnknownFields) { - auto *issue = ev->Record.AddIssues(); - issue->set_severity(errorLevel); - issue->set_message(TStringBuilder{} << "Unknown key# " << info.first << " in proto# " << info.second << " found in path# " << path); - } - - for (auto& [path, info] : opCtx.DeprecatedFields) { - auto *issue = ev->Record.AddIssues(); - issue->set_severity(NYql::TSeverityIds::S_WARNING); - issue->set_message(TStringBuilder{} << "Deprecated key# " << info.first << " in proto# " << info.second << " found in path# " << path); - } - - Response = MakeHolder(Sender, ctx.SelfID, ev.Release()); - }; - Version = opCtx.Version; // UpdatedConfig = opCtx.UpdatedConfig; Cluster = opCtx.Cluster; @@ -316,7 +300,8 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig // 4) support dry-run // 5) support audit auto ev = MakeHolder(); - fillResponse(ev, NYql::TSeverityIds::S_WARNING); + Response = FillResponse(opCtx, ev, NYql::TSeverityIds::S_WARNING, ctx); + return true; // if (!DryRun && !hasForbiddenUnknown) { // DoInternalAudit(txc, ctx); @@ -338,13 +323,13 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig auto ev = MakeHolder(); ev->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); ErrorReason = "Unknown keys in config."; - fillResponse(ev, NYql::TSeverityIds::S_ERROR); + Response = FillResponse(opCtx, ev, NYql::TSeverityIds::S_ERROR, ctx); } else if (!Force) { auto ev = MakeHolder(); - fillResponse(ev, NYql::TSeverityIds::S_WARNING); + Response = FillResponse(opCtx, ev, NYql::TSeverityIds::S_WARNING, ctx); } else { auto ev = MakeHolder(); - fillResponse(ev, NYql::TSeverityIds::S_WARNING); + Response = FillResponse(opCtx, ev, NYql::TSeverityIds::S_WARNING, ctx); } } catch (const yexception& ex) { From 538b67dd1deb30a3c487cb0f3cd0919e80d93e23 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Thu, 30 Jan 2025 13:03:42 +0000 Subject: [PATCH 18/54] WIP --- .../console/console__replace_yaml_config.cpp | 70 ++++++++++++------- ydb/core/protos/console_config.proto | 3 +- 2 files changed, 47 insertions(+), 26 deletions(-) diff --git a/ydb/core/cms/console/console__replace_yaml_config.cpp b/ydb/core/cms/console/console__replace_yaml_config.cpp index 1c7f6228d458..066f60a46c6d 100644 --- a/ydb/core/cms/console/console__replace_yaml_config.cpp +++ b/ydb/core/cms/console/console__replace_yaml_config.cpp @@ -64,26 +64,6 @@ class TConfigsManager::TTxReplaceYamlConfigBase return MakeHolder(Sender, ctx.SelfID, ev.Release()); } - void DoInternalAudit(TTransactionContext &txc, const TActorContext &ctx) - { - auto logData = NKikimrConsole::TLogRecordData{}; - - // for backward compatibility in ui - logData.MutableAction()->AddActions()->MutableModifyConfigItem()->MutableConfigItem(); - logData.AddAffectedKinds(NKikimrConsole::TConfigItem::YamlConfigChangeItem); - - auto& yamlConfigChange = *logData.MutableYamlConfigChange(); - yamlConfigChange.SetOldYamlConfig(Self->YamlConfig); - // yamlConfigChange.SetNewYamlConfig(UpdatedConfig); // FIXME - for (auto& [id, config] : Self->VolatileYamlConfigs) { - auto& oldVolatileConfig = *yamlConfigChange.AddOldVolatileYamlConfigs(); - oldVolatileConfig.SetId(id); - oldVolatileConfig.SetConfig(config); - } - - Self->Logger.DbLogData(UserToken.GetUserSID(), logData, txc, ctx); - } - void HandleError(const TString& error, const TActorContext& ctx) { Error = true; auto ev = MakeHolder(); @@ -230,6 +210,26 @@ class TConfigsManager::TTxReplaceMainYamlConfig Self->TxProcessor->TxCompleted(this, ctx); } + void DoInternalAudit(TTransactionContext &txc, const TActorContext &ctx) + { + auto logData = NKikimrConsole::TLogRecordData{}; + + // for backward compatibility in ui + logData.MutableAction()->AddActions()->MutableModifyConfigItem()->MutableConfigItem(); + logData.AddAffectedKinds(NKikimrConsole::TConfigItem::YamlConfigChangeItem); + + auto& yamlConfigChange = *logData.MutableYamlConfigChange(); + yamlConfigChange.SetOldYamlConfig(Self->YamlConfig); + yamlConfigChange.SetNewYamlConfig(UpdatedConfig); + for (auto& [id, config] : Self->VolatileYamlConfigs) { + auto& oldVolatileConfig = *yamlConfigChange.AddOldVolatileYamlConfigs(); + oldVolatileConfig.SetId(id); + oldVolatileConfig.SetConfig(config); + } + + Self->Logger.DbLogData(UserToken.GetUserSID(), logData, txc, ctx); + } + private: TString UpdatedConfig; }; @@ -257,8 +257,8 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig NIceDb::TNiceDb db(txc.DB); TUpdateConfigOpContext opCtx; - Self->ReplaceMainConfigMetadata(Config, false, opCtx); - Self->ValidateMainConfig(opCtx); + Self->ReplaceMainConfigMetadata(Config, false, opCtx); // FIXME + Self->ValidateMainConfig(opCtx); // FIXME bool hasForbiddenUnknown = !opCtx.UnknownFields.empty() && !AllowUnknownFields; if (opCtx.Error) { @@ -303,8 +303,8 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig Response = FillResponse(opCtx, ev, NYql::TSeverityIds::S_WARNING, ctx); return true; - // if (!DryRun && !hasForbiddenUnknown) { - // DoInternalAudit(txc, ctx); + if (!DryRun && !hasForbiddenUnknown) { + DoInternalAudit(txc, ctx); // db.Table().Key(Version + 1) // .Update(UpdatedConfig) @@ -316,7 +316,7 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig // /* Later we shift this boundary to support rollback and history */ // db.Table().Key(Version) // .Delete(); - // } + } if (hasForbiddenUnknown) { Error = true; @@ -339,6 +339,26 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig return true; } + void DoInternalAudit(TTransactionContext &txc, const TActorContext &ctx) + { + auto logData = NKikimrConsole::TLogRecordData{}; + + // for backward compatibility in ui + logData.MutableAction()->AddActions()->MutableModifyConfigItem()->MutableConfigItem(); + logData.AddAffectedKinds(NKikimrConsole::TConfigItem::DatabaseYamlConfigChangeItem); + + // auto& yamlConfigChange = *logData.MutableYamlConfigChange(); + // yamlConfigChange.SetOldYamlConfig(Self->YamlConfig); + // yamlConfigChange.SetNewYamlConfig(UpdatedConfig); + // for (auto& [id, config] : Self->VolatileYamlConfigs) { + // auto& oldVolatileConfig = *yamlConfigChange.AddOldVolatileYamlConfigs(); + // oldVolatileConfig.SetId(id); + // oldVolatileConfig.SetConfig(config); + // } + + Self->Logger.DbLogData(UserToken.GetUserSID(), logData, txc, ctx); + } + void Complete(const TActorContext &ctx) override { Y_UNUSED(ctx); // TODO diff --git a/ydb/core/protos/console_config.proto b/ydb/core/protos/console_config.proto index 2515842ec7a2..46b5a95098de 100644 --- a/ydb/core/protos/console_config.proto +++ b/ydb/core/protos/console_config.proto @@ -147,7 +147,8 @@ message TConfigItem { NamedConfigsItem = 100; ClusterYamlConfigItem = 101; - // synthetic kind for audit purposes only + // synthetic kinds for audit purposes only + DatabaseYamlConfigChangeItem = 32767; YamlConfigChangeItem = 32768; } From 5bd5df7ab81eb5030902c8837b1b1ab446b96eb4 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Thu, 30 Jan 2025 13:19:34 +0000 Subject: [PATCH 19/54] WIP --- .../console/console__replace_yaml_config.cpp | 30 ++++++++----------- ydb/core/protos/console.proto | 7 +++++ 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/ydb/core/cms/console/console__replace_yaml_config.cpp b/ydb/core/cms/console/console__replace_yaml_config.cpp index 066f60a46c6d..6b09a2990dec 100644 --- a/ydb/core/cms/console/console__replace_yaml_config.cpp +++ b/ydb/core/cms/console/console__replace_yaml_config.cpp @@ -29,7 +29,7 @@ class TConfigsManager::TTxReplaceYamlConfigBase , Force(force) , AllowUnknownFields(ev->Get()->Record.GetRequest().allow_unknown_fields()) , DryRun(ev->Get()->Record.GetRequest().dry_run()) - , Database(ev->Get()->Record.HasDatabase() ? TMaybe{ev->Get()->Record.GetDatabase()} : TMaybe{}) + , InputDatabase(ev->Get()->Record.HasDatabase() ? TMaybe{ev->Get()->Record.GetDatabase()} : TMaybe{}) { } @@ -90,8 +90,8 @@ class TConfigsManager::TTxReplaceYamlConfigBase TSimpleSharedPtr UnknownFieldsCollector = nullptr; ui32 Version; TString Cluster; - TMaybe Database; - bool WarnDatabaseByPass = false; + TMaybe InputDatabase; + bool WarnDatabaseBypass = false; }; class TConfigsManager::TTxReplaceMainYamlConfig @@ -132,8 +132,8 @@ class TConfigsManager::TTxReplaceMainYamlConfig Cluster = opCtx.Cluster; Modify = opCtx.UpdatedConfig != Self->YamlConfig || Self->YamlDropped; - if (Database) { - WarnDatabaseByPass = true; + if (InputDatabase) { + WarnDatabaseBypass = true; } if (!DryRun && !hasForbiddenUnknown) { @@ -272,8 +272,8 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig Cluster = opCtx.Cluster; // Modify = opCtx.UpdatedConfig != Self->YamlConfig || Self->YamlDropped; - // if (!Database) { FIXME: extract database from config itself - // WarnDatabaseByPass = true; + // if (!InputDatabase) { FIXME: extract database from config itself + // WarnDatabaseBypass = true; // } if (!AppData(ctx)->FeatureFlags.GetPerDatabaseConfigAllowed()) { @@ -289,7 +289,7 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig return true; } - Self->YamlConfigPerDatabase[*Database] = Config; + Self->YamlConfigPerDatabase[*InputDatabase] = Config; // FIXME Modify = true; // TODO @@ -347,14 +347,10 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig logData.MutableAction()->AddActions()->MutableModifyConfigItem()->MutableConfigItem(); logData.AddAffectedKinds(NKikimrConsole::TConfigItem::DatabaseYamlConfigChangeItem); - // auto& yamlConfigChange = *logData.MutableYamlConfigChange(); - // yamlConfigChange.SetOldYamlConfig(Self->YamlConfig); - // yamlConfigChange.SetNewYamlConfig(UpdatedConfig); - // for (auto& [id, config] : Self->VolatileYamlConfigs) { - // auto& oldVolatileConfig = *yamlConfigChange.AddOldVolatileYamlConfigs(); - // oldVolatileConfig.SetId(id); - // oldVolatileConfig.SetConfig(config); - // } + auto& databaseConfigChange = *logData.MutableDatabaseConfigChange(); + databaseConfigChange.SetDatabase(TargetDatabase); + databaseConfigChange.SetOldYamlConfig("old"); + databaseConfigChange.SetNewYamlConfig("new"); Self->Logger.DbLogData(UserToken.GetUserSID(), logData, txc, ctx); } @@ -365,7 +361,7 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig } private: - // TODO + TString TargetDatabase; }; ITransaction *TConfigsManager::CreateTxReplaceMainYamlConfig(TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev) diff --git a/ydb/core/protos/console.proto b/ydb/core/protos/console.proto index e6d83427afc5..b7ca6afdd1f8 100644 --- a/ydb/core/protos/console.proto +++ b/ydb/core/protos/console.proto @@ -38,6 +38,12 @@ message TYamlConfigChange { repeated TVolatileConfig NewVolatileYamlConfigs = 4; } +message TDatabaseConfigChange { + optional string Database = 1; + optional string OldYamlConfig = 2; + optional string NewYamlConfig = 3; +} + message TLogRecordData { // Old configs optional TConfigureRequest Action = 1; @@ -45,6 +51,7 @@ message TLogRecordData { repeated uint32 AffectedKinds = 3; // New configs optional TYamlConfigChange YamlConfigChange = 4; + optional TDatabaseConfigChange DatabaseConfigChange = 5; } message TLogRecord { From b9a70fed1c94039cc776f524640999d31083b932 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Thu, 30 Jan 2025 14:06:33 +0000 Subject: [PATCH 20/54] WIP --- ydb/core/cms/console/console__get_yaml_config.cpp | 2 +- ydb/core/cms/console/console__replace_yaml_config.cpp | 5 ++++- ydb/core/cms/console/console_configs_manager.h | 2 +- ydb/core/cms/console/console_configs_provider.cpp | 2 +- ydb/core/cms/console/console_configs_provider.h | 11 ++++++++--- 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/ydb/core/cms/console/console__get_yaml_config.cpp b/ydb/core/cms/console/console__get_yaml_config.cpp index 6f491ddff82d..c99c36d98faa 100644 --- a/ydb/core/cms/console/console__get_yaml_config.cpp +++ b/ydb/core/cms/console/console__get_yaml_config.cpp @@ -22,7 +22,7 @@ class TConfigsManager::TTxGetYamlConfig : public TTransactionBaseYamlConfigPerDatabase.contains(*Database)) { - Response->Record.MutableResponse()->set_config(Self->YamlConfigPerDatabase[*Database]); + Response->Record.MutableResponse()->set_config(Self->YamlConfigPerDatabase[*Database].Config); } return true; diff --git a/ydb/core/cms/console/console__replace_yaml_config.cpp b/ydb/core/cms/console/console__replace_yaml_config.cpp index 6b09a2990dec..2ff88526bbf9 100644 --- a/ydb/core/cms/console/console__replace_yaml_config.cpp +++ b/ydb/core/cms/console/console__replace_yaml_config.cpp @@ -289,7 +289,10 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig return true; } - Self->YamlConfigPerDatabase[*InputDatabase] = Config; + Self->YamlConfigPerDatabase[*InputDatabase] = TDatabaseYamlConfig { + .Config = Config, + .Version = Version, + }; // FIXME Modify = true; // TODO diff --git a/ydb/core/cms/console/console_configs_manager.h b/ydb/core/cms/console/console_configs_manager.h index 8971a1f2a78c..f23c72e8d2e1 100644 --- a/ydb/core/cms/console/console_configs_manager.h +++ b/ydb/core/cms/console/console_configs_manager.h @@ -305,7 +305,7 @@ class TConfigsManager : public TActorBootstrapped { TString ClusterName; ui32 YamlVersion = 0; TString YamlConfig; - THashMap YamlConfigPerDatabase; + THashMap YamlConfigPerDatabase; bool YamlDropped = false; bool YamlReadOnly = true; TMap VolatileYamlConfigs; diff --git a/ydb/core/cms/console/console_configs_provider.cpp b/ydb/core/cms/console/console_configs_provider.cpp index 216f3afb6fe3..69b36a88c65f 100644 --- a/ydb/core/cms/console/console_configs_provider.cpp +++ b/ydb/core/cms/console/console_configs_provider.cpp @@ -1266,7 +1266,7 @@ void TConfigsProvider::Handle(TEvPrivate::TEvUpdateYamlConfig::TPtr &ev, const T subscription->VolatileYamlConfigHashes = VolatileYamlConfigHashes; if (YamlConfigPerDatabase.contains(subscription->Tenant)) { - request->Record.SetDatabaseConfig(YamlConfigPerDatabase[subscription->Tenant]); + request->Record.SetDatabaseConfig(YamlConfigPerDatabase[subscription->Tenant].Config); } ctx.Send(subscription->Worker, request.Release()); diff --git a/ydb/core/cms/console/console_configs_provider.h b/ydb/core/cms/console/console_configs_provider.h index b230d7fa3ddc..57888b05f8e2 100644 --- a/ydb/core/cms/console/console_configs_provider.h +++ b/ydb/core/cms/console/console_configs_provider.h @@ -13,6 +13,11 @@ namespace NKikimr::NConsole { +struct TDatabaseYamlConfig { + TString Config; + ui32 Version; +}; + class TConfigsProvider : public TActorBootstrapped { public: struct TEvPrivate { @@ -110,14 +115,14 @@ class TConfigsProvider : public TActorBootstrapped { TEvUpdateYamlConfig( const TString &yamlConfig, - const THashMap &yamlConfigPerDatabase) + const THashMap &yamlConfigPerDatabase) : YamlConfig(yamlConfig) , YamlConfigPerDatabase(yamlConfigPerDatabase) { } TString YamlConfig; - THashMap YamlConfigPerDatabase; + THashMap YamlConfigPerDatabase; TMap VolatileYamlConfigs; }; @@ -240,7 +245,7 @@ class TConfigsProvider : public TActorBootstrapped { TString YamlConfig; TMap VolatileYamlConfigs; - THashMap YamlConfigPerDatabase; + THashMap YamlConfigPerDatabase; ui64 YamlConfigVersion = 0; TMap VolatileYamlConfigHashes; }; From 5e95ae475046f60b1e0e10c56bc1ddd8adb2f7f4 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 12:46:27 +0000 Subject: [PATCH 21/54] WIP --- .../console/console__replace_yaml_config.cpp | 39 ++++------- .../cms/console/console_configs_manager.cpp | 66 +++++++++++++++++++ .../cms/console/console_configs_manager.h | 22 ++++++- 3 files changed, 99 insertions(+), 28 deletions(-) diff --git a/ydb/core/cms/console/console__replace_yaml_config.cpp b/ydb/core/cms/console/console__replace_yaml_config.cpp index 2ff88526bbf9..0b685e0ef550 100644 --- a/ydb/core/cms/console/console__replace_yaml_config.cpp +++ b/ydb/core/cms/console/console__replace_yaml_config.cpp @@ -48,7 +48,7 @@ class TConfigsManager::TTxReplaceYamlConfigBase { } - THolder FillResponse(const TUpdateConfigOpContext& opCtx, auto& ev, auto errorLevel, const TActorContext &ctx) { + THolder FillResponse(const TUpdateConfigOpBaseContext& opCtx, auto& ev, auto errorLevel, const TActorContext &ctx) { for (auto& [path, info] : opCtx.UnknownFields) { auto *issue = ev->Record.AddIssues(); issue->set_severity(errorLevel); @@ -89,7 +89,6 @@ class TConfigsManager::TTxReplaceYamlConfigBase bool Modify = false; TSimpleSharedPtr UnknownFieldsCollector = nullptr; ui32 Version; - TString Cluster; TMaybe InputDatabase; bool WarnDatabaseBypass = false; }; @@ -231,6 +230,7 @@ class TConfigsManager::TTxReplaceMainYamlConfig } private: + TString Cluster; TString UpdatedConfig; }; @@ -256,9 +256,9 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig { NIceDb::TNiceDb db(txc.DB); - TUpdateConfigOpContext opCtx; - Self->ReplaceMainConfigMetadata(Config, false, opCtx); // FIXME - Self->ValidateMainConfig(opCtx); // FIXME + TUpdateDatabaseConfigOpContext opCtx; + Self->ReplaceDatabaseConfigMetadata(Config, false, opCtx); + Self->ValidateDatabaseConfig(opCtx); bool hasForbiddenUnknown = !opCtx.UnknownFields.empty() && !AllowUnknownFields; if (opCtx.Error) { @@ -268,9 +268,8 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig try { Version = opCtx.Version; - // UpdatedConfig = opCtx.UpdatedConfig; - Cluster = opCtx.Cluster; - // Modify = opCtx.UpdatedConfig != Self->YamlConfig || Self->YamlDropped; + UpdatedConfig = opCtx.UpdatedConfig; + // Modify = opCtx.UpdatedConfig != Self->YamlConfig; // if (!InputDatabase) { FIXME: extract database from config itself // WarnDatabaseBypass = true; @@ -295,13 +294,6 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig }; // FIXME Modify = true; - // TODO - // 1) send - // 1) persist - // 2) validate - // 3) support force - // 4) support dry-run - // 5) support audit auto ev = MakeHolder(); Response = FillResponse(opCtx, ev, NYql::TSeverityIds::S_WARNING, ctx); return true; @@ -309,16 +301,12 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig if (!DryRun && !hasForbiddenUnknown) { DoInternalAudit(txc, ctx); - // db.Table().Key(Version + 1) - // .Update(UpdatedConfig) - // // set config dropped by default to support rollback to previous versions - // // where new config layout is not supported - // // it will lead to ignoring config from new versions - // .Update(true); + db.Table().Key(TargetDatabase, Version + 1) + .Update(Config); - // /* Later we shift this boundary to support rollback and history */ - // db.Table().Key(Version) - // .Delete(); + /* Later we shift this boundary to support rollback and history */ + db.Table().Key(TargetDatabase, Version) + .Delete(); } if (hasForbiddenUnknown) { @@ -360,11 +348,12 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig void Complete(const TActorContext &ctx) override { - Y_UNUSED(ctx); // TODO + Self->TxProcessor->TxCompleted(this, ctx); } private: TString TargetDatabase; + TString UpdatedConfig; }; ITransaction *TConfigsManager::CreateTxReplaceMainYamlConfig(TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev) diff --git a/ydb/core/cms/console/console_configs_manager.cpp b/ydb/core/cms/console/console_configs_manager.cpp index c727862f503e..c9f16fc49787 100644 --- a/ydb/core/cms/console/console_configs_manager.cpp +++ b/ydb/core/cms/console/console_configs_manager.cpp @@ -116,6 +116,72 @@ void TConfigsManager::ValidateMainConfig(TUpdateConfigOpContext& opCtx) { } } +void TConfigsManager::ReplaceDatabaseConfigMetadata(const TString &config, bool force, TUpdateDatabaseConfigOpContext& opCtx) { + // FIXME + try { + if (!force) { + auto metadata = NYamlConfig::GetMetadata(config); + opCtx.Cluster = metadata.Cluster.value_or(TString("unknown")); + opCtx.Version = metadata.Version.value_or(0); + } else { + opCtx.Cluster = ClusterName; + opCtx.Version = YamlVersion; + } + + opCtx.UpdatedConfig = NYamlConfig::ReplaceMetadata(config, NYamlConfig::TMainMetadata{ + .Version = opCtx.Version + 1, + .Cluster = opCtx.Cluster, + }); + } catch (const yexception &e) { + opCtx.Error = e.what(); + } +} + +void TConfigsManager::ValidateDatabaseConfig(TUpdateDatabaseConfigOpContext& opCtx) { + // FIXME + try { + if (opCtx.UpdatedConfig != YamlConfig || YamlDropped) { + auto tree = NFyaml::TDocument::Parse(opCtx.UpdatedConfig); + auto resolved = NYamlConfig::ResolveAll(tree); + + if (ClusterName != opCtx.Cluster) { + ythrow yexception() << "ClusterName mismatch"; + } + + if (opCtx.Version != YamlVersion) { + ythrow yexception() << "Version mismatch"; + } + + TSimpleSharedPtr unknownFieldsCollector = new NYamlConfig::TBasicUnknownFieldsCollector; + + std::vector errors; + for (auto& [_, config] : resolved.Configs) { + auto cfg = NYamlConfig::YamlToProto( + config.second, + true, + true, + unknownFieldsCollector); + NKikimr::NConfig::EValidationResult result = NKikimr::NConfig::ValidateConfig(cfg, errors); + if (result == NKikimr::NConfig::EValidationResult::Error) { + ythrow yexception() << errors.front(); + } + } + + const auto& deprecatedPaths = NKikimrConfig::TAppConfig::GetReservedChildrenPaths(); + + for (const auto& [path, info] : unknownFieldsCollector->GetUnknownKeys()) { + if (deprecatedPaths.contains(path)) { + opCtx.DeprecatedFields[path] = info; + } else { + opCtx.UnknownFields[path] = info; + } + } + } + } catch (const yexception &e) { + opCtx.Error = e.what(); + } +} + void TConfigsManager::Bootstrap(const TActorContext &ctx) { LOG_DEBUG(ctx, NKikimrServices::CMS_CONFIGS, "TConfigsManager::Bootstrap"); diff --git a/ydb/core/cms/console/console_configs_manager.h b/ydb/core/cms/console/console_configs_manager.h index f23c72e8d2e1..8ad486c5f0e2 100644 --- a/ydb/core/cms/console/console_configs_manager.h +++ b/ydb/core/cms/console/console_configs_manager.h @@ -32,13 +32,26 @@ class TConfigsManager : public TActorBootstrapped { using TBase = TActorBootstrapped; - struct TUpdateConfigOpContext { + struct TUpdateConfigOpBaseContext { std::optional Error; + + TMap> DeprecatedFields; + TMap> UnknownFields; + }; + + struct TUpdateConfigOpContext + : public TUpdateConfigOpBaseContext + { TString UpdatedConfig; ui32 Version; TString Cluster; - TMap> DeprecatedFields; - TMap> UnknownFields; + }; + + struct TUpdateDatabaseConfigOpContext + : public TUpdateConfigOpBaseContext + { + TString UpdatedConfig; + ui32 Version; }; public: @@ -71,6 +84,9 @@ class TConfigsManager : public TActorBootstrapped { void ReplaceMainConfigMetadata(const TString &config, bool force, TUpdateConfigOpContext& opCtx); void ValidateMainConfig(TUpdateConfigOpContext& opCtx); + void ReplaceDatabaseConfigMetadata(const TString &config, bool force, TUpdateDatabaseConfigOpContext& opCtx); + void ValidateDatabaseConfig(TUpdateDatabaseConfigOpContext& opCtx); + void SendInReply(const TActorId& sender, const TActorId& icSession, std::unique_ptr ev, ui64 cookie = 0); void ApplyPendingConfigModifications(const TActorContext &ctx, From b7c07abe5e9defc5b764b94d7e44448dcb62fc79 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 13:15:35 +0000 Subject: [PATCH 22/54] WIP --- .../cms/console/console_configs_manager.cpp | 20 +++++++++---------- .../cms/console/console_configs_manager.h | 1 + .../yaml_config/public/yaml_config.cpp | 11 ++++++++++ ydb/library/yaml_config/public/yaml_config.h | 5 +++++ 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/ydb/core/cms/console/console_configs_manager.cpp b/ydb/core/cms/console/console_configs_manager.cpp index c9f16fc49787..08c54bf82329 100644 --- a/ydb/core/cms/console/console_configs_manager.cpp +++ b/ydb/core/cms/console/console_configs_manager.cpp @@ -117,20 +117,24 @@ void TConfigsManager::ValidateMainConfig(TUpdateConfigOpContext& opCtx) { } void TConfigsManager::ReplaceDatabaseConfigMetadata(const TString &config, bool force, TUpdateDatabaseConfigOpContext& opCtx) { - // FIXME try { + auto metadata = NYamlConfig::GetDatabaseMetadata(config); + + if (!metadata.Database) { + ythrow yexception() << "metadata.database is not present, unable to infer target database"; + } + + opCtx.TargetDatabase = *metadata.Database; + if (!force) { - auto metadata = NYamlConfig::GetMetadata(config); - opCtx.Cluster = metadata.Cluster.value_or(TString("unknown")); opCtx.Version = metadata.Version.value_or(0); } else { - opCtx.Cluster = ClusterName; opCtx.Version = YamlVersion; } - opCtx.UpdatedConfig = NYamlConfig::ReplaceMetadata(config, NYamlConfig::TMainMetadata{ + opCtx.UpdatedConfig = NYamlConfig::ReplaceMetadata(config, NYamlConfig::TDatabaseMetadata{ .Version = opCtx.Version + 1, - .Cluster = opCtx.Cluster, + .Database = opCtx.TargetDatabase, }); } catch (const yexception &e) { opCtx.Error = e.what(); @@ -144,10 +148,6 @@ void TConfigsManager::ValidateDatabaseConfig(TUpdateDatabaseConfigOpContext& opC auto tree = NFyaml::TDocument::Parse(opCtx.UpdatedConfig); auto resolved = NYamlConfig::ResolveAll(tree); - if (ClusterName != opCtx.Cluster) { - ythrow yexception() << "ClusterName mismatch"; - } - if (opCtx.Version != YamlVersion) { ythrow yexception() << "Version mismatch"; } diff --git a/ydb/core/cms/console/console_configs_manager.h b/ydb/core/cms/console/console_configs_manager.h index 8ad486c5f0e2..fa744f88f133 100644 --- a/ydb/core/cms/console/console_configs_manager.h +++ b/ydb/core/cms/console/console_configs_manager.h @@ -51,6 +51,7 @@ class TConfigsManager : public TActorBootstrapped { : public TUpdateConfigOpBaseContext { TString UpdatedConfig; + TString TargetDatabase; ui32 Version; }; diff --git a/ydb/library/yaml_config/public/yaml_config.cpp b/ydb/library/yaml_config/public/yaml_config.cpp index 293e61d7f3cb..55a30f0d58ff 100644 --- a/ydb/library/yaml_config/public/yaml_config.cpp +++ b/ydb/library/yaml_config/public/yaml_config.cpp @@ -732,6 +732,17 @@ TString ReplaceMetadata(const TString& config, const TMainMetadata& metadata) { return ReplaceMetadata(config, serializeMetadata); } +TString ReplaceMetadata(const TString& config, const TDatabaseMetadata& metadata) { + auto serializeMetadata = [&](TStringStream& sstr) { + sstr + << "metadata:" + << "\n kind: DatabaseConfig" + << "\n database: \"" << *metadata.Database << "\"" + << "\n version: " << *metadata.Version; + }; + return ReplaceMetadata(config, serializeMetadata); +} + TString ReplaceMetadata(const TString& config, const TVolatileMetadata& metadata) { auto serializeMetadata = [&](TStringStream& sstr) { sstr diff --git a/ydb/library/yaml_config/public/yaml_config.h b/ydb/library/yaml_config/public/yaml_config.h index 21e9eed763fa..cb7ccf62adb7 100644 --- a/ydb/library/yaml_config/public/yaml_config.h +++ b/ydb/library/yaml_config/public/yaml_config.h @@ -222,6 +222,11 @@ TVolatileMetadata GetVolatileMetadata(const TString& config); */ TString ReplaceMetadata(const TString& config, const TMainMetadata& metadata); +/** + * Replaces metadata in database config + */ +TString ReplaceMetadata(const TString& config, const TDatabaseMetadata& metadata); + /** * Replaces volatile metadata in config */ From a3c65425542875ce3f1d0dedbc934815b21d3b48 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 13:22:24 +0000 Subject: [PATCH 23/54] WIP --- ydb/core/cms/console/console_configs_manager.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ydb/core/cms/console/console_configs_manager.cpp b/ydb/core/cms/console/console_configs_manager.cpp index 08c54bf82329..fcf6dc46b242 100644 --- a/ydb/core/cms/console/console_configs_manager.cpp +++ b/ydb/core/cms/console/console_configs_manager.cpp @@ -754,6 +754,7 @@ void TConfigsManager::Handle(TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev, TxProcessor->ProcessTx(CreateTxReplaceMainYamlConfig(ev), ctx); }, [&](const NYamlConfig::TDatabaseMetadata& /* value */) { + // TODO: validate database exists && user has rights to change it TxProcessor->ProcessTx(CreateTxReplaceDatabaseYamlConfig(ev), ctx); }, [&](auto&...) { @@ -777,6 +778,7 @@ void TConfigsManager::Handle(TEvConsole::TEvSetYamlConfigRequest::TPtr &ev, cons TxProcessor->ProcessTx(CreateTxSetMainYamlConfig(ev), ctx); }, [&](const NYamlConfig::TDatabaseMetadata& /* value */) { + // TODO: validate database exists && user has rights to change it TxProcessor->ProcessTx(CreateTxSetDatabaseYamlConfig(ev), ctx); }, [&](auto&...) { From a20f2da31cedea87d3874e8379becba2cec0e95d Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 13:33:54 +0000 Subject: [PATCH 24/54] WIP --- .../console/console__replace_yaml_config.cpp | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/ydb/core/cms/console/console__replace_yaml_config.cpp b/ydb/core/cms/console/console__replace_yaml_config.cpp index 0b685e0ef550..39b0577ccd14 100644 --- a/ydb/core/cms/console/console__replace_yaml_config.cpp +++ b/ydb/core/cms/console/console__replace_yaml_config.cpp @@ -29,7 +29,7 @@ class TConfigsManager::TTxReplaceYamlConfigBase , Force(force) , AllowUnknownFields(ev->Get()->Record.GetRequest().allow_unknown_fields()) , DryRun(ev->Get()->Record.GetRequest().dry_run()) - , InputDatabase(ev->Get()->Record.HasDatabase() ? TMaybe{ev->Get()->Record.GetDatabase()} : TMaybe{}) + , IngressDatabase(ev->Get()->Record.HasDatabase() ? TMaybe{ev->Get()->Record.GetDatabase()} : TMaybe{}) { } @@ -88,8 +88,7 @@ class TConfigsManager::TTxReplaceYamlConfigBase TString ErrorReason; bool Modify = false; TSimpleSharedPtr UnknownFieldsCollector = nullptr; - ui32 Version; - TMaybe InputDatabase; + TMaybe IngressDatabase; bool WarnDatabaseBypass = false; }; @@ -131,7 +130,7 @@ class TConfigsManager::TTxReplaceMainYamlConfig Cluster = opCtx.Cluster; Modify = opCtx.UpdatedConfig != Self->YamlConfig || Self->YamlDropped; - if (InputDatabase) { + if (IngressDatabase) { WarnDatabaseBypass = true; } @@ -230,6 +229,7 @@ class TConfigsManager::TTxReplaceMainYamlConfig } private: + ui32 Version; TString Cluster; TString UpdatedConfig; }; @@ -271,9 +271,9 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig UpdatedConfig = opCtx.UpdatedConfig; // Modify = opCtx.UpdatedConfig != Self->YamlConfig; - // if (!InputDatabase) { FIXME: extract database from config itself - // WarnDatabaseBypass = true; - // } + if (IngressDatabase != opCtx.TargetDatabase) { + WarnDatabaseBypass = true; + } if (!AppData(ctx)->FeatureFlags.GetPerDatabaseConfigAllowed()) { Error = true; @@ -288,15 +288,14 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig return true; } - Self->YamlConfigPerDatabase[*InputDatabase] = TDatabaseYamlConfig { + Self->YamlConfigPerDatabase[*IngressDatabase] = TDatabaseYamlConfig { .Config = Config, .Version = Version, }; - // FIXME - Modify = true; + + // FIXME ? auto ev = MakeHolder(); Response = FillResponse(opCtx, ev, NYql::TSeverityIds::S_WARNING, ctx); - return true; if (!DryRun && !hasForbiddenUnknown) { DoInternalAudit(txc, ctx); @@ -352,6 +351,7 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig } private: + ui32 Version; TString TargetDatabase; TString UpdatedConfig; }; From e0753b0279d82bd26d150a728d061572e0ba10fe Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 13:53:15 +0000 Subject: [PATCH 25/54] WIP --- .../console/console__replace_yaml_config.cpp | 37 ++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/ydb/core/cms/console/console__replace_yaml_config.cpp b/ydb/core/cms/console/console__replace_yaml_config.cpp index 39b0577ccd14..43af8bff55a4 100644 --- a/ydb/core/cms/console/console__replace_yaml_config.cpp +++ b/ydb/core/cms/console/console__replace_yaml_config.cpp @@ -288,11 +288,6 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig return true; } - Self->YamlConfigPerDatabase[*IngressDatabase] = TDatabaseYamlConfig { - .Config = Config, - .Version = Version, - }; - // FIXME ? auto ev = MakeHolder(); Response = FillResponse(opCtx, ev, NYql::TSeverityIds::S_WARNING, ctx); @@ -347,6 +342,38 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig void Complete(const TActorContext &ctx) override { + LOG_DEBUG(ctx, NKikimrServices::CMS_CONFIGS, "TTxReplaceYamlConfig Complete"); + + ctx.Send(Response.Release()); + + if (!Error && Modify && !DryRun) { + // AuditLogReplaceConfigTransaction( + // /* peer = */ Peer, + // /* userSID = */ UserToken.GetUserSID(), + // /* sanitizedToken = */ UserToken.GetSanitizedToken(), + // /* oldConfig = */ Self->YamlConfig, + // /* newConfig = */ Config, + // /* reason = */ {}, + // /* success = */ true); + + Self->YamlConfigPerDatabase[*IngressDatabase] = TDatabaseYamlConfig { + .Config = UpdatedConfig, + .Version = Version + 1, + }; + + auto resp = MakeHolder(Self->YamlConfig, Self->YamlConfigPerDatabase); + ctx.Send(Self->ConfigsProvider, resp.Release()); + } else if (Error && !DryRun) { + // AuditLogReplaceConfigTransaction( + // /* peer = */ Peer, + // /* userSID = */ UserToken.GetUserSID(), + // /* sanitizedToken = */ UserToken.GetSanitizedToken(), + // /* oldConfig = */ Self->YamlConfig, + // /* newConfig = */ Config, + // /* reason = */ ErrorReason, + // /* success = */ false); + } + Self->TxProcessor->TxCompleted(this, ctx); } From e3e5d3424671b6a09930be97b9bb9b37406b6d64 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 14:29:23 +0000 Subject: [PATCH 26/54] WIP --- ydb/core/cms/console/console.cpp | 9 +++++ .../cms/console/console_configs_manager.cpp | 40 ++++++++++--------- .../cms/console/console_configs_manager.h | 2 + ydb/core/cms/console/console_impl.h | 2 + .../cms/console/console_tenants_manager.cpp | 4 +- .../cms/console/console_tenants_manager.h | 7 +++- 6 files changed, 42 insertions(+), 22 deletions(-) diff --git a/ydb/core/cms/console/console.cpp b/ydb/core/cms/console/console.cpp index 6c856ea785e5..64e39beabbaf 100644 --- a/ydb/core/cms/console/console.cpp +++ b/ydb/core/cms/console/console.cpp @@ -186,6 +186,15 @@ void TConsole::Handle(TEvConsole::TEvSetConfigRequest::TPtr &ev, const TActorCon TxProcessor->ProcessTx(CreateTxSetConfig(ev), ctx); } +bool TConsole::HasTenant(const TString& path) const +{ + if (!TenantsManager) { + return false; + } + + return TenantsManager->HasTenant(path); +} + IActor *CreateConsole(const TActorId &tablet, TTabletStorageInfo *info) { return new TConsole(tablet, info); diff --git a/ydb/core/cms/console/console_configs_manager.cpp b/ydb/core/cms/console/console_configs_manager.cpp index fcf6dc46b242..6a42c9ba4bda 100644 --- a/ydb/core/cms/console/console_configs_manager.cpp +++ b/ydb/core/cms/console/console_configs_manager.cpp @@ -753,18 +753,15 @@ void TConfigsManager::Handle(TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev, [&](const NYamlConfig::TMainMetadata& /* value */) { TxProcessor->ProcessTx(CreateTxReplaceMainYamlConfig(ev), ctx); }, - [&](const NYamlConfig::TDatabaseMetadata& /* value */) { - // TODO: validate database exists && user has rights to change it + [&](const NYamlConfig::TDatabaseMetadata& value) { + if (!value.Database || !Self.HasTenant(*value.Database)) { + // TODO fail + return; + } TxProcessor->ProcessTx(CreateTxReplaceDatabaseYamlConfig(ev), ctx); }, [&](auto&...) { - auto resp = MakeHolder(); - resp->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); - auto *issue = resp->Record.AddIssues(); - issue->set_severity(NYql::TSeverityIds::S_ERROR); - issue->set_message("Unknown config kind or format"); - auto response = MakeHolder(ev->Sender, ctx.SelfID, resp.Release()); - ctx.Send(response.Release()); + return FailReplaceConfig(ev->Sender, "Unknown config kind or format", ctx); } }, metadata); } @@ -777,22 +774,29 @@ void TConfigsManager::Handle(TEvConsole::TEvSetYamlConfigRequest::TPtr &ev, cons [&](const NYamlConfig::TMainMetadata& /* value */) { TxProcessor->ProcessTx(CreateTxSetMainYamlConfig(ev), ctx); }, - [&](const NYamlConfig::TDatabaseMetadata& /* value */) { - // TODO: validate database exists && user has rights to change it + [&](const NYamlConfig::TDatabaseMetadata& value) { + if (!value.Database || !Self.HasTenant(*value.Database)) { + // TODO fail + return; + } TxProcessor->ProcessTx(CreateTxSetDatabaseYamlConfig(ev), ctx); }, [&](auto&...) { - auto resp = MakeHolder(); - resp->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); - auto *issue = resp->Record.AddIssues(); - issue->set_severity(NYql::TSeverityIds::S_ERROR); - issue->set_message("Unknown config kind or format"); - auto response = MakeHolder(ev->Sender, ctx.SelfID, resp.Release()); - ctx.Send(response.Release()); + return FailReplaceConfig(ev->Sender, "Unknown config kind or format", ctx); } }, metadata); } +void TConfigsManager::FailReplaceConfig(TActorId Sender, const TString& error, const TActorContext &ctx) { + auto resp = MakeHolder(); + resp->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); + auto *issue = resp->Record.AddIssues(); + issue->set_severity(NYql::TSeverityIds::S_ERROR); + issue->set_message(error); + auto response = MakeHolder(Sender, ctx.SelfID, resp.Release()); + ctx.Send(response.Release()); +} + void TConfigsManager::Handle(TEvConsole::TEvDropConfigRequest::TPtr &ev, const TActorContext &ctx) { TxProcessor->ProcessTx(CreateTxDropYamlConfig(ev), ctx); diff --git a/ydb/core/cms/console/console_configs_manager.h b/ydb/core/cms/console/console_configs_manager.h index fa744f88f133..5763f096960c 100644 --- a/ydb/core/cms/console/console_configs_manager.h +++ b/ydb/core/cms/console/console_configs_manager.h @@ -202,6 +202,8 @@ class TConfigsManager : public TActorBootstrapped { void Handle(TEvBlobStorage::TEvControllerConsoleCommitRequest::TPtr &ev, const TActorContext &ctx); void Handle(TEvBlobStorage::TEvControllerValidateConfigRequest::TPtr &ev, const TActorContext &ctx); + void FailReplaceConfig(TActorId Sender, const TString& error, const TActorContext &ctx); + static bool CheckRights(const TString& userToken); template diff --git a/ydb/core/cms/console/console_impl.h b/ydb/core/cms/console/console_impl.h index eb4432ca0eb6..a5de7325048f 100644 --- a/ydb/core/cms/console/console_impl.h +++ b/ydb/core/cms/console/console_impl.h @@ -179,6 +179,8 @@ class TConsole : public TActor return Config; } + bool HasTenant(const TString& path) const; + private: TDeque> InitQueue; NKikimrConsole::TConfig Config; diff --git a/ydb/core/cms/console/console_tenants_manager.cpp b/ydb/core/cms/console/console_tenants_manager.cpp index bca83aa8e8a7..e0e2f6e00639 100644 --- a/ydb/core/cms/console/console_tenants_manager.cpp +++ b/ydb/core/cms/console/console_tenants_manager.cpp @@ -1627,7 +1627,7 @@ TTenantsManager::TTenant::TPtr TTenantsManager::FindComputationalUnitKindUsage(c return nullptr; } -TTenantsManager::TTenant::TPtr TTenantsManager::GetTenant(const TString &name) +TTenantsManager::TTenant::TPtr TTenantsManager::GetTenant(const TString &name) const { auto it = Tenants.find(name); if (it != Tenants.end()) @@ -1635,7 +1635,7 @@ TTenantsManager::TTenant::TPtr TTenantsManager::GetTenant(const TString &name) return nullptr; } -TTenantsManager::TTenant::TPtr TTenantsManager::GetTenant(const TDomainId &domainId) +TTenantsManager::TTenant::TPtr TTenantsManager::GetTenant(const TDomainId &domainId) const { auto it = TenantIdToName.find(domainId); if (it != TenantIdToName.end()) diff --git a/ydb/core/cms/console/console_tenants_manager.h b/ydb/core/cms/console/console_tenants_manager.h index e1af8f391f15..04c92558821b 100644 --- a/ydb/core/cms/console/console_tenants_manager.h +++ b/ydb/core/cms/console/console_tenants_manager.h @@ -756,8 +756,8 @@ class TTenantsManager : public TActorBootstrapped { TTenant::TPtr FindComputationalUnitKindUsage(const TString &kind); TTenant::TPtr FindComputationalUnitKindUsage(const TString &kind, const TString &zone); - TTenant::TPtr GetTenant(const TString &name); - TTenant::TPtr GetTenant(const TDomainId &domainId); + TTenant::TPtr GetTenant(const TString &name) const; + TTenant::TPtr GetTenant(const TDomainId &domainId) const; void AddTenant(TTenant::TPtr tenant); void RemoveTenant(TTenant::TPtr tenant); void RemoveTenantFailed(TTenant::TPtr tenant, @@ -1006,6 +1006,9 @@ class TTenantsManager : public TActorBootstrapped { void Bootstrap(const TActorContext &ctx); void Detach(); + bool HasTenant(const TString& path) const { + return Tenants.contains(path); + } private: TConsole &Self; From 390a60d2db90ac318fd6866a6180fc453340cc35 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 14:45:11 +0000 Subject: [PATCH 27/54] WIP --- ydb/core/cms/console/console_configs_manager.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ydb/core/cms/console/console_configs_manager.cpp b/ydb/core/cms/console/console_configs_manager.cpp index 6a42c9ba4bda..9251dc5d5da4 100644 --- a/ydb/core/cms/console/console_configs_manager.cpp +++ b/ydb/core/cms/console/console_configs_manager.cpp @@ -755,8 +755,7 @@ void TConfigsManager::Handle(TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev, }, [&](const NYamlConfig::TDatabaseMetadata& value) { if (!value.Database || !Self.HasTenant(*value.Database)) { - // TODO fail - return; + return FailReplaceConfig(ev->Sender, "Unknown database", ctx); } TxProcessor->ProcessTx(CreateTxReplaceDatabaseYamlConfig(ev), ctx); }, @@ -776,8 +775,7 @@ void TConfigsManager::Handle(TEvConsole::TEvSetYamlConfigRequest::TPtr &ev, cons }, [&](const NYamlConfig::TDatabaseMetadata& value) { if (!value.Database || !Self.HasTenant(*value.Database)) { - // TODO fail - return; + return FailReplaceConfig(ev->Sender, "Unknown database", ctx); } TxProcessor->ProcessTx(CreateTxSetDatabaseYamlConfig(ev), ctx); }, @@ -788,7 +786,8 @@ void TConfigsManager::Handle(TEvConsole::TEvSetYamlConfigRequest::TPtr &ev, cons } void TConfigsManager::FailReplaceConfig(TActorId Sender, const TString& error, const TActorContext &ctx) { - auto resp = MakeHolder(); + // TODO: +audit + auto resp = MakeHolder(); resp->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); auto *issue = resp->Record.AddIssues(); issue->set_severity(NYql::TSeverityIds::S_ERROR); From 2592a3b309f096742de588a0cf7ed95128f109d3 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 14:49:55 +0000 Subject: [PATCH 28/54] WIP --- ydb/core/cms/console/console_configs_subscriber.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/ydb/core/cms/console/console_configs_subscriber.cpp b/ydb/core/cms/console/console_configs_subscriber.cpp index 0d8362f7146f..dfe5ad9b857d 100644 --- a/ydb/core/cms/console/console_configs_subscriber.cpp +++ b/ydb/core/cms/console/console_configs_subscriber.cpp @@ -361,7 +361,6 @@ class TConfigsSubscriber : public TActorBootstrapped { request->Record.SetGeneration(Generation = NextGeneration++); request->Record.MutableOptions()->SetNodeId(SelfId().NodeId()); - Cerr << " xxx 2 " << Tenant << Endl; request->Record.MutableOptions()->SetTenant(Tenant); request->Record.MutableOptions()->SetNodeType(NodeType); request->Record.MutableOptions()->SetHost(FQDNHostName()); From ffe8668b7b5bf13116e205953f663b5f5d3ab02c Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 15:28:35 +0000 Subject: [PATCH 29/54] WIP --- .../console/console__replace_yaml_config.cpp | 50 ++++++++++++------- ydb/core/cms/console/console_audit.cpp | 31 ++++++++++-- ydb/core/cms/console/console_audit.h | 10 ++++ 3 files changed, 70 insertions(+), 21 deletions(-) diff --git a/ydb/core/cms/console/console__replace_yaml_config.cpp b/ydb/core/cms/console/console__replace_yaml_config.cpp index 43af8bff55a4..038917fe2cbb 100644 --- a/ydb/core/cms/console/console__replace_yaml_config.cpp +++ b/ydb/core/cms/console/console__replace_yaml_config.cpp @@ -326,6 +326,12 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig void DoInternalAudit(TTransactionContext &txc, const TActorContext &ctx) { + TString oldConfig; + + if (Self->YamlConfigPerDatabase.contains(TargetDatabase)) { + oldConfig = Self->YamlConfigPerDatabase[TargetDatabase].Config; + } + auto logData = NKikimrConsole::TLogRecordData{}; // for backward compatibility in ui @@ -334,8 +340,8 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig auto& databaseConfigChange = *logData.MutableDatabaseConfigChange(); databaseConfigChange.SetDatabase(TargetDatabase); - databaseConfigChange.SetOldYamlConfig("old"); - databaseConfigChange.SetNewYamlConfig("new"); + databaseConfigChange.SetOldYamlConfig(oldConfig); + databaseConfigChange.SetNewYamlConfig(UpdatedConfig); Self->Logger.DbLogData(UserToken.GetUserSID(), logData, txc, ctx); } @@ -346,15 +352,22 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig ctx.Send(Response.Release()); + TString oldConfig; + + if (Self->YamlConfigPerDatabase.contains(TargetDatabase)) { + oldConfig = Self->YamlConfigPerDatabase[TargetDatabase].Config; + } + if (!Error && Modify && !DryRun) { - // AuditLogReplaceConfigTransaction( - // /* peer = */ Peer, - // /* userSID = */ UserToken.GetUserSID(), - // /* sanitizedToken = */ UserToken.GetSanitizedToken(), - // /* oldConfig = */ Self->YamlConfig, - // /* newConfig = */ Config, - // /* reason = */ {}, - // /* success = */ true); + AuditLogReplaceDatabaseConfigTransaction( + /* peer = */ Peer, + /* userSID = */ UserToken.GetUserSID(), + /* sanitizedToken = */ UserToken.GetSanitizedToken(), + /* database = */ TargetDatabase, + /* oldConfig = */ oldConfig, + /* newConfig = */ Config, + /* reason = */ {}, + /* success = */ true); Self->YamlConfigPerDatabase[*IngressDatabase] = TDatabaseYamlConfig { .Config = UpdatedConfig, @@ -364,14 +377,15 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig auto resp = MakeHolder(Self->YamlConfig, Self->YamlConfigPerDatabase); ctx.Send(Self->ConfigsProvider, resp.Release()); } else if (Error && !DryRun) { - // AuditLogReplaceConfigTransaction( - // /* peer = */ Peer, - // /* userSID = */ UserToken.GetUserSID(), - // /* sanitizedToken = */ UserToken.GetSanitizedToken(), - // /* oldConfig = */ Self->YamlConfig, - // /* newConfig = */ Config, - // /* reason = */ ErrorReason, - // /* success = */ false); + AuditLogReplaceDatabaseConfigTransaction( + /* peer = */ Peer, + /* userSID = */ UserToken.GetUserSID(), + /* sanitizedToken = */ UserToken.GetSanitizedToken(), + /* database = */ TargetDatabase, + /* oldConfig = */ oldConfig, + /* newConfig = */ Config, + /* reason = */ ErrorReason, + /* success = */ false); } Self->TxProcessor->TxCompleted(this, ctx); diff --git a/ydb/core/cms/console/console_audit.cpp b/ydb/core/cms/console/console_audit.cpp index 68a70bb62799..e2b6d8e8b89d 100644 --- a/ydb/core/cms/console/console_audit.cpp +++ b/ydb/core/cms/console/console_audit.cpp @@ -5,6 +5,9 @@ namespace NKikimr::NConsole { +static const TString COMPONENT_NAME = "console"; +static const TString EMPTY_VALUE = "{none}"; + void AuditLogReplaceConfigTransaction( const TString& peer, const TString& userSID, @@ -14,10 +17,31 @@ void AuditLogReplaceConfigTransaction( const TString& reason, bool success) { - static const TString COMPONENT_NAME = "console"; + auto peerName = NKikimr::NAddressClassifier::ExtractAddress(peer); - static const TString EMPTY_VALUE = "{none}"; + AUDIT_LOG( + AUDIT_PART("component", COMPONENT_NAME) + AUDIT_PART("remote_address", (!peerName.empty() ? peerName : EMPTY_VALUE)) + AUDIT_PART("subject", (!userSID.empty() ? userSID : EMPTY_VALUE)) + AUDIT_PART("sanitized_token", (!sanitizedToken.empty() ? sanitizedToken : EMPTY_VALUE)) + AUDIT_PART("status", TString(success ? "SUCCESS" : "ERROR")) + AUDIT_PART("reason", reason, !reason.empty()) + AUDIT_PART("operation", TString("REPLACE DYNCONFIG")) + AUDIT_PART("old_config", oldConfig) + AUDIT_PART("new_config", newConfig) + ); +} +void AuditLogReplaceDatabaseConfigTransaction( + const TString& peer, + const TString& userSID, + const TString& sanitizedToken, + const TString& database, + const TString& oldConfig, + const TString& newConfig, + const TString& reason, + bool success) +{ auto peerName = NKikimr::NAddressClassifier::ExtractAddress(peer); AUDIT_LOG( @@ -25,9 +49,10 @@ void AuditLogReplaceConfigTransaction( AUDIT_PART("remote_address", (!peerName.empty() ? peerName : EMPTY_VALUE)) AUDIT_PART("subject", (!userSID.empty() ? userSID : EMPTY_VALUE)) AUDIT_PART("sanitized_token", (!sanitizedToken.empty() ? sanitizedToken : EMPTY_VALUE)) + AUDIT_PART("database", (!database.empty() ? database : EMPTY_VALUE)) AUDIT_PART("status", TString(success ? "SUCCESS" : "ERROR")) AUDIT_PART("reason", reason, !reason.empty()) - AUDIT_PART("operation", TString("REPLACE DYNCONFIG")) + AUDIT_PART("operation", TString("REPLACE DATABASE CONFIG")) AUDIT_PART("old_config", oldConfig) AUDIT_PART("new_config", newConfig) ); diff --git a/ydb/core/cms/console/console_audit.h b/ydb/core/cms/console/console_audit.h index b3c94e37e1fc..77a55db7837f 100644 --- a/ydb/core/cms/console/console_audit.h +++ b/ydb/core/cms/console/console_audit.h @@ -13,4 +13,14 @@ void AuditLogReplaceConfigTransaction( const TString& reason, bool success); +void AuditLogReplaceDatabaseConfigTransaction( + const TString& peer, + const TString& userSID, + const TString& sanitizedToken, + const TString& database, + const TString& oldConfig, + const TString& newConfig, + const TString& reason, + bool success); + } // namespace NKikimr::NConsole From 651e918e089276eaa324c4d7daab989b0b283cd8 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 16:36:59 +0000 Subject: [PATCH 30/54] WIP --- .../cms/console/console_configs_manager.cpp | 24 +++++++++++++++---- .../yaml_config/public/yaml_config.cpp | 10 +++++--- ydb/library/yaml_config/public/yaml_config.h | 6 ++++- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/ydb/core/cms/console/console_configs_manager.cpp b/ydb/core/cms/console/console_configs_manager.cpp index 9251dc5d5da4..52742b786a36 100644 --- a/ydb/core/cms/console/console_configs_manager.cpp +++ b/ydb/core/cms/console/console_configs_manager.cpp @@ -759,8 +759,16 @@ void TConfigsManager::Handle(TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev, } TxProcessor->ProcessTx(CreateTxReplaceDatabaseYamlConfig(ev), ctx); }, - [&](auto&...) { - return FailReplaceConfig(ev->Sender, "Unknown config kind or format", ctx); + [&](const NYamlConfig::TError& error) { + AuditLogReplaceConfigTransaction( + /* peer = */ ev->Get()->Record.GetPeerName(), + /* userSID = */ NACLib::TUserToken(ev->Get()->Record.GetUserToken()).GetUserSID(), + /* sanitizedToken = */ NACLib::TUserToken(ev->Get()->Record.GetUserToken()).GetSanitizedToken(), + /* oldConfig = */ YamlConfig, + /* newConfig = */ ev->Get()->Record.GetRequest().config(), + /* reason = */ error.Error, + /* success = */ false); + return FailReplaceConfig(ev->Sender, error.Error, ctx); } }, metadata); } @@ -779,8 +787,16 @@ void TConfigsManager::Handle(TEvConsole::TEvSetYamlConfigRequest::TPtr &ev, cons } TxProcessor->ProcessTx(CreateTxSetDatabaseYamlConfig(ev), ctx); }, - [&](auto&...) { - return FailReplaceConfig(ev->Sender, "Unknown config kind or format", ctx); + [&](const NYamlConfig::TError& error) { + AuditLogReplaceConfigTransaction( + /* peer = */ ev->Get()->Record.GetPeerName(), + /* userSID = */ NACLib::TUserToken(ev->Get()->Record.GetUserToken()).GetUserSID(), + /* sanitizedToken = */ NACLib::TUserToken(ev->Get()->Record.GetUserToken()).GetSanitizedToken(), + /* oldConfig = */ YamlConfig, + /* newConfig = */ ev->Get()->Record.GetRequest().config(), + /* reason = */ error.Error, + /* success = */ false); + return FailReplaceConfig(ev->Sender, error.Error, ctx); } }, metadata); } diff --git a/ydb/library/yaml_config/public/yaml_config.cpp b/ydb/library/yaml_config/public/yaml_config.cpp index 55a30f0d58ff..4979c77de676 100644 --- a/ydb/library/yaml_config/public/yaml_config.cpp +++ b/ydb/library/yaml_config/public/yaml_config.cpp @@ -798,7 +798,7 @@ TString StripMetadata(const TString& config) { return sstr.Str(); } -std::variant GetGenericMetadata(const TString& config) { +std::variant GetGenericMetadata(const TString& config) { try { auto doc = NFyaml::TDocument::Parse(config); auto kind = doc.Root().Map().at("metadata").Map().at("kind").Scalar(); @@ -807,10 +807,14 @@ std::variant GetGenericMetadat } else if (kind == "DatabaseConfig") { return GetDatabaseMetadata(config); } else { - return {}; + return TError { + .Error = TString("Unknown kind: ") + kind, + }; } } catch (yexception& e) { - return {}; + return TError { + .Error = e.what(), + }; } } diff --git a/ydb/library/yaml_config/public/yaml_config.h b/ydb/library/yaml_config/public/yaml_config.h index cb7ccf62adb7..7136cd6244e7 100644 --- a/ydb/library/yaml_config/public/yaml_config.h +++ b/ydb/library/yaml_config/public/yaml_config.h @@ -197,10 +197,14 @@ struct TDatabaseMetadata { std::optional Database; }; +struct TError { + TString Error; +}; + /** * Parses config metadata */ -std::variant GetGenericMetadata(const TString& config); +std::variant GetGenericMetadata(const TString& config); /** * Parses config metadata From 17ba81b3d1616c706d250c191b75dc4ee88fd2c1 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 17:15:10 +0000 Subject: [PATCH 31/54] WIP --- ydb/tests/functional/audit/test_auditlog.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ydb/tests/functional/audit/test_auditlog.py b/ydb/tests/functional/audit/test_auditlog.py index 166db3a50b2b..f4f8d4534cf4 100644 --- a/ydb/tests/functional/audit/test_auditlog.py +++ b/ydb/tests/functional/audit/test_auditlog.py @@ -434,7 +434,8 @@ def test_dynconfig(ydb_cluster, prepared_test_env, _client_session_pool_with_aut _client_session_pool_with_auth_root.retry_operation_sync(apply_config, config=config) print(capture_audit.captured, file=sys.stderr) - assert json.dumps(config) in capture_audit.captured + cfg = json.dumps(config) + assert cfg in capture_audit.captured @pytest.mark.parametrize('config_fixture', ["_bad_dynconfig", "_good_dynconfig"]) @@ -450,4 +451,5 @@ def test_broken_dynconfig(ydb_cluster, prepared_test_env, pool_fixture, config_f pass print(capture_audit.captured, file=sys.stderr) - assert json.dumps(config) in capture_audit.captured + cfg = json.dumps(config) + assert cfg in capture_audit.captured From b70c9b265a6e6a72418d34d5c3b104de619d0d79 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 18:07:47 +0000 Subject: [PATCH 32/54] WIP --- .../console/console__replace_yaml_config.cpp | 6 +----- .../cms/console/console_configs_manager.cpp | 21 ++++++++++++++++++- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/ydb/core/cms/console/console__replace_yaml_config.cpp b/ydb/core/cms/console/console__replace_yaml_config.cpp index 038917fe2cbb..102dcd3545ca 100644 --- a/ydb/core/cms/console/console__replace_yaml_config.cpp +++ b/ydb/core/cms/console/console__replace_yaml_config.cpp @@ -288,15 +288,11 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig return true; } - // FIXME ? - auto ev = MakeHolder(); - Response = FillResponse(opCtx, ev, NYql::TSeverityIds::S_WARNING, ctx); - if (!DryRun && !hasForbiddenUnknown) { DoInternalAudit(txc, ctx); db.Table().Key(TargetDatabase, Version + 1) - .Update(Config); + .Update(Config); /* Later we shift this boundary to support rollback and history */ db.Table().Key(TargetDatabase, Version) diff --git a/ydb/core/cms/console/console_configs_manager.cpp b/ydb/core/cms/console/console_configs_manager.cpp index 52742b786a36..aae0e168758c 100644 --- a/ydb/core/cms/console/console_configs_manager.cpp +++ b/ydb/core/cms/console/console_configs_manager.cpp @@ -455,14 +455,18 @@ bool TConfigsManager::DbLoadState(TTransactionContext &txc, auto subscriptionRowset = db.Table().Range().Select(); auto validatorsRowset = db.Table().Range().Select(); auto yamlConfigRowset = db.Table().Reverse().Select(); + auto perDatabaseYamlConfigRowset = db.Table().Select(); if (!configItemRowset.IsReady() || !nextConfigItemIdRow.IsReady() || !nextSubscriptionIdRow.IsReady() || !subscriptionRowset.IsReady() || !validatorsRowset.IsReady() - || !yamlConfigRowset.IsReady()) + || !yamlConfigRowset.IsReady() + || !perDatabaseYamlConfigRowset.IsReady()) + { return false; + } if (nextConfigItemIdRow.IsValid()) { TString value = nextConfigItemIdRow.GetValue(); @@ -496,6 +500,21 @@ bool TConfigsManager::DbLoadState(TTransactionContext &txc, YamlDropped = false; } + while (!perDatabaseYamlConfigRowset.EndOfSet()) { + TString tenant = perDatabaseYamlConfigRowset.GetValue(); + ui32 version = perDatabaseYamlConfigRowset.GetValue(); + TString config = perDatabaseYamlConfigRowset.GetValue(); + + YamlConfigPerDatabase[tenant] = TDatabaseYamlConfig { + .Config = config, + .Version = version, + }; + + if (!perDatabaseYamlConfigRowset.Next()) { + return false; + } + } + while (!configItemRowset.EndOfSet()) { ui64 id = configItemRowset.GetValue(); ui64 generation = configItemRowset.GetValue(); From c326bffaa4b3591b298ca0dba950b9d1b1e31dcc Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 18:21:57 +0000 Subject: [PATCH 33/54] WIP --- ydb/core/cms/console/console_configs_manager.cpp | 2 +- ydb/core/cms/console/console_configs_provider.h | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/ydb/core/cms/console/console_configs_manager.cpp b/ydb/core/cms/console/console_configs_manager.cpp index aae0e168758c..6b9d0dcbda90 100644 --- a/ydb/core/cms/console/console_configs_manager.cpp +++ b/ydb/core/cms/console/console_configs_manager.cpp @@ -1129,7 +1129,7 @@ void TConfigsManager::Handle(TEvPrivate::TEvStateLoaded::TPtr &/*ev*/, const TAc ctx.Send(ConfigsProvider, new TConfigsProvider::TEvPrivate::TEvSetSubscriptions(SubscriptionIndex.GetSubscriptions())); ctx.Send(GetNameserviceActorId(), new TEvInterconnect::TEvListNodes()); if (!YamlConfig.empty()) { - ctx.Send(ConfigsProvider, new TConfigsProvider::TEvPrivate::TEvUpdateYamlConfig(YamlConfig, VolatileYamlConfigs)); + ctx.Send(ConfigsProvider, new TConfigsProvider::TEvPrivate::TEvUpdateYamlConfig(YamlConfig, YamlConfigPerDatabase, VolatileYamlConfigs)); } ScheduleLogCleanup(ctx); } diff --git a/ydb/core/cms/console/console_configs_provider.h b/ydb/core/cms/console/console_configs_provider.h index 57888b05f8e2..350089e17dc8 100644 --- a/ydb/core/cms/console/console_configs_provider.h +++ b/ydb/core/cms/console/console_configs_provider.h @@ -121,6 +121,16 @@ class TConfigsProvider : public TActorBootstrapped { { } + TEvUpdateYamlConfig( + const TString &yamlConfig, + const THashMap &yamlConfigPerDatabase, + const TMap &volatileYamlConfigs) + : YamlConfig(yamlConfig) + , YamlConfigPerDatabase(yamlConfigPerDatabase) + , VolatileYamlConfigs(volatileYamlConfigs) + { + } + TString YamlConfig; THashMap YamlConfigPerDatabase; TMap VolatileYamlConfigs; From ec81445393b107e8a8bfa13fca2570c935ced45f Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 18:34:52 +0000 Subject: [PATCH 34/54] WIP --- .../cms/console/console__replace_yaml_config.cpp | 7 ++++++- ydb/core/cms/console/console_configs_provider.h | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/ydb/core/cms/console/console__replace_yaml_config.cpp b/ydb/core/cms/console/console__replace_yaml_config.cpp index 102dcd3545ca..ed88565f8733 100644 --- a/ydb/core/cms/console/console__replace_yaml_config.cpp +++ b/ydb/core/cms/console/console__replace_yaml_config.cpp @@ -370,7 +370,12 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig .Version = Version + 1, }; - auto resp = MakeHolder(Self->YamlConfig, Self->YamlConfigPerDatabase); + auto resp = MakeHolder( + Self->YamlConfig, + Self->YamlConfigPerDatabase, + Self->VolatileYamlConfigs, + TargetDatabase); + ctx.Send(Self->ConfigsProvider, resp.Release()); } else if (Error && !DryRun) { AuditLogReplaceDatabaseConfigTransaction( diff --git a/ydb/core/cms/console/console_configs_provider.h b/ydb/core/cms/console/console_configs_provider.h index 350089e17dc8..99e7b5b83187 100644 --- a/ydb/core/cms/console/console_configs_provider.h +++ b/ydb/core/cms/console/console_configs_provider.h @@ -131,9 +131,22 @@ class TConfigsProvider : public TActorBootstrapped { { } + TEvUpdateYamlConfig( + const TString &yamlConfig, + const THashMap &yamlConfigPerDatabase, + const TMap &volatileYamlConfigs, + const TString& changedDatabase) + : YamlConfig(yamlConfig) + , YamlConfigPerDatabase(yamlConfigPerDatabase) + , VolatileYamlConfigs(volatileYamlConfigs) + , ChangedDatabase(changedDatabase) + { + } + TString YamlConfig; THashMap YamlConfigPerDatabase; TMap VolatileYamlConfigs; + TString ChangedDatabase; }; struct TEvUpdateSubscriptions : public TEventLocal { From 82452e6ab38c5cc8966afd54a19c65c4b181a0aa Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 18:58:05 +0000 Subject: [PATCH 35/54] WIP --- .../cms/console/console_configs_provider.cpp | 85 ++++++++++--------- ydb/core/cms/console/util/config_index.cpp | 9 ++ ydb/core/cms/console/util/config_index.h | 1 + 3 files changed, 55 insertions(+), 40 deletions(-) diff --git a/ydb/core/cms/console/console_configs_provider.cpp b/ydb/core/cms/console/console_configs_provider.cpp index 69b36a88c65f..49f0f985066d 100644 --- a/ydb/core/cms/console/console_configs_provider.cpp +++ b/ydb/core/cms/console/console_configs_provider.cpp @@ -1223,54 +1223,59 @@ void TConfigsProvider::Handle(TEvPrivate::TEvUpdateSubscriptions::TPtr &ev, cons } void TConfigsProvider::Handle(TEvPrivate::TEvUpdateYamlConfig::TPtr &ev, const TActorContext &ctx) { - YamlConfig = ev->Get()->YamlConfig; YamlConfigPerDatabase = ev->Get()->YamlConfigPerDatabase; - VolatileYamlConfigs.clear(); - - YamlConfigVersion = NYamlConfig::GetVersion(YamlConfig); - VolatileYamlConfigHashes.clear(); - - for (auto& [id, config] : ev->Get()->VolatileYamlConfigs) { - auto doc = NFyaml::TDocument::Parse(config); - // we strip it to provide old format for config dispatcher - auto node = doc.Root().Map().at("selector_config"); - TString strippedConfig = "\n" + config.substr(node.BeginMark().InputPos, node.EndMark().InputPos - node.BeginMark().InputPos) + "\n"; - VolatileYamlConfigs[id] = strippedConfig; - VolatileYamlConfigHashes[id] = THash()(strippedConfig); - } - - for (auto &[_, subscription] : InMemoryIndex.GetSubscriptions()) { - if (subscription->ServeYaml) { - auto request = MakeHolder( - subscription->Generation, - NKikimrConfig::TAppConfig{}, - THashSet{}); - - if (subscription->YamlConfigVersion != YamlConfigVersion) { - subscription->YamlConfigVersion = YamlConfigVersion; - request->Record.SetYamlConfig(YamlConfig); - } else { - request->Record.SetYamlConfigNotChanged(true); - } + if (!ev->Get()->ChangedDatabase) { + YamlConfig = ev->Get()->YamlConfig; + VolatileYamlConfigs.clear(); + + YamlConfigVersion = NYamlConfig::GetVersion(YamlConfig); + VolatileYamlConfigHashes.clear(); + + for (auto& [id, config] : ev->Get()->VolatileYamlConfigs) { + auto doc = NFyaml::TDocument::Parse(config); + // we strip it to provide old format for config dispatcher + auto node = doc.Root().Map().at("selector_config"); + TString strippedConfig = "\n" + config.substr(node.BeginMark().InputPos, node.EndMark().InputPos - node.BeginMark().InputPos) + "\n"; + VolatileYamlConfigs[id] = strippedConfig; + VolatileYamlConfigHashes[id] = THash()(strippedConfig); + } + + for (auto &[_, subscription] : InMemoryIndex.GetSubscriptions()) { + if (subscription->ServeYaml) { + auto request = MakeHolder( + subscription->Generation, + NKikimrConfig::TAppConfig{}, + THashSet{}); - for (auto &[id, hash] : VolatileYamlConfigHashes) { - auto *volatileConfig = request->Record.AddVolatileConfigs(); - volatileConfig->SetId(id); - if (auto it = subscription->VolatileYamlConfigHashes.find(id); it != subscription->VolatileYamlConfigHashes.end() && it->second == hash) { - volatileConfig->SetNotChanged(true); + if (subscription->YamlConfigVersion != YamlConfigVersion) { + subscription->YamlConfigVersion = YamlConfigVersion; + request->Record.SetYamlConfig(YamlConfig); } else { - volatileConfig->SetConfig(VolatileYamlConfigs[id]); + request->Record.SetYamlConfigNotChanged(true); } - } - subscription->VolatileYamlConfigHashes = VolatileYamlConfigHashes; + for (auto &[id, hash] : VolatileYamlConfigHashes) { + auto *volatileConfig = request->Record.AddVolatileConfigs(); + volatileConfig->SetId(id); + if (auto it = subscription->VolatileYamlConfigHashes.find(id); it != subscription->VolatileYamlConfigHashes.end() && it->second == hash) { + volatileConfig->SetNotChanged(true); + } else { + volatileConfig->SetConfig(VolatileYamlConfigs[id]); + } + } - if (YamlConfigPerDatabase.contains(subscription->Tenant)) { - request->Record.SetDatabaseConfig(YamlConfigPerDatabase[subscription->Tenant].Config); - } + subscription->VolatileYamlConfigHashes = VolatileYamlConfigHashes; + + if (YamlConfigPerDatabase.contains(subscription->Tenant)) { + request->Record.SetDatabaseConfig(YamlConfigPerDatabase[subscription->Tenant].Config); + } - ctx.Send(subscription->Worker, request.Release()); + ctx.Send(subscription->Worker, request.Release()); + } } + } else { + const auto* subs = InMemoryIndex.GetSubscriptions(ev->Get()->ChangedDatabase); + Y_UNUSED(subs); } } diff --git a/ydb/core/cms/console/util/config_index.cpp b/ydb/core/cms/console/util/config_index.cpp index 75d620fbaa1a..57dd50e63ddb 100644 --- a/ydb/core/cms/console/util/config_index.cpp +++ b/ydb/core/cms/console/util/config_index.cpp @@ -701,6 +701,15 @@ const THashMap &TInMemorySubscriptionInde return Subscriptions; } +const TInMemorySubscriptionSet *TInMemorySubscriptionIndex::GetSubscriptions(const TString& path) const +{ + if (auto it = SubscriptionsByTenant.find(path); it != SubscriptionsByTenant.end()) { + return &(it->second); + } + + return nullptr; +} + void TInMemorySubscriptionIndex::AddSubscription(TInMemorySubscription::TPtr subscription) { Y_ABORT_UNLESS(!Subscriptions.contains(subscription->Subscriber)); diff --git a/ydb/core/cms/console/util/config_index.h b/ydb/core/cms/console/util/config_index.h index 821f9482bd8e..3a6d56c336a5 100644 --- a/ydb/core/cms/console/util/config_index.h +++ b/ydb/core/cms/console/util/config_index.h @@ -684,6 +684,7 @@ class TInMemorySubscriptionIndex { TInMemorySubscription::TPtr GetSubscription(const TActorId &subscriber); const THashMap &GetSubscriptions() const; + const TInMemorySubscriptionSet *GetSubscriptions(const TString &path) const; void AddSubscription(TInMemorySubscription::TPtr subscription); From 9cec7237e067a2d8daec77da3419d075487108bf Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 19:32:38 +0000 Subject: [PATCH 36/54] WIP --- .../cms/console/console_configs_provider.cpp | 73 +++++++++++-------- .../cms/console/console_configs_provider.h | 3 + 2 files changed, 45 insertions(+), 31 deletions(-) diff --git a/ydb/core/cms/console/console_configs_provider.cpp b/ydb/core/cms/console/console_configs_provider.cpp index 49f0f985066d..bd62c95667bb 100644 --- a/ydb/core/cms/console/console_configs_provider.cpp +++ b/ydb/core/cms/console/console_configs_provider.cpp @@ -1241,42 +1241,53 @@ void TConfigsProvider::Handle(TEvPrivate::TEvUpdateYamlConfig::TPtr &ev, const T } for (auto &[_, subscription] : InMemoryIndex.GetSubscriptions()) { - if (subscription->ServeYaml) { - auto request = MakeHolder( - subscription->Generation, - NKikimrConfig::TAppConfig{}, - THashSet{}); - - if (subscription->YamlConfigVersion != YamlConfigVersion) { - subscription->YamlConfigVersion = YamlConfigVersion; - request->Record.SetYamlConfig(YamlConfig); - } else { - request->Record.SetYamlConfigNotChanged(true); - } - - for (auto &[id, hash] : VolatileYamlConfigHashes) { - auto *volatileConfig = request->Record.AddVolatileConfigs(); - volatileConfig->SetId(id); - if (auto it = subscription->VolatileYamlConfigHashes.find(id); it != subscription->VolatileYamlConfigHashes.end() && it->second == hash) { - volatileConfig->SetNotChanged(true); - } else { - volatileConfig->SetConfig(VolatileYamlConfigs[id]); - } - } + UpdateConfig(subscription, ctx); + } + } else { + const auto* subs = InMemoryIndex.GetSubscriptions(ev->Get()->ChangedDatabase); + if (subs) { + for (auto &subscription : *subs) { + UpdateConfig(subscription, ctx); + } + } + } +} - subscription->VolatileYamlConfigHashes = VolatileYamlConfigHashes; +void TConfigsProvider::UpdateConfig(TInMemorySubscription::TPtr subscription, + const TActorContext &ctx) +{ + if (subscription->ServeYaml) { + auto request = MakeHolder( + subscription->Generation, + NKikimrConfig::TAppConfig{}, + THashSet{}); - if (YamlConfigPerDatabase.contains(subscription->Tenant)) { - request->Record.SetDatabaseConfig(YamlConfigPerDatabase[subscription->Tenant].Config); - } + if (subscription->YamlConfigVersion != YamlConfigVersion) { + subscription->YamlConfigVersion = YamlConfigVersion; + request->Record.SetYamlConfig(YamlConfig); + } else { + request->Record.SetYamlConfigNotChanged(true); + } - ctx.Send(subscription->Worker, request.Release()); - } + for (auto &[id, hash] : VolatileYamlConfigHashes) { + auto *volatileConfig = request->Record.AddVolatileConfigs(); + volatileConfig->SetId(id); + if (auto it = subscription->VolatileYamlConfigHashes.find(id); it != subscription->VolatileYamlConfigHashes.end() && it->second == hash) { + volatileConfig->SetNotChanged(true); + } else { + volatileConfig->SetConfig(VolatileYamlConfigs[id]); } - } else { - const auto* subs = InMemoryIndex.GetSubscriptions(ev->Get()->ChangedDatabase); - Y_UNUSED(subs); } + + subscription->VolatileYamlConfigHashes = VolatileYamlConfigHashes; + + if (YamlConfigPerDatabase.contains(subscription->Tenant)) { + request->Record.SetDatabaseConfig(YamlConfigPerDatabase[subscription->Tenant].Config); + } + + ctx.Send(subscription->Worker, request.Release()); +} + } } // namespace NKikimr::NConsole diff --git a/ydb/core/cms/console/console_configs_provider.h b/ydb/core/cms/console/console_configs_provider.h index 99e7b5b83187..a6a45f02c87e 100644 --- a/ydb/core/cms/console/console_configs_provider.h +++ b/ydb/core/cms/console/console_configs_provider.h @@ -181,6 +181,9 @@ class TConfigsProvider : public TActorBootstrapped { void CheckSubscription(TInMemorySubscription::TPtr subscriptions, const TActorContext &ctx); + void UpdateConfig(TInMemorySubscription::TPtr subscription, + const TActorContext &ctx); + void Handle(NMon::TEvHttpInfo::TPtr &ev); void Handle(TEvConsole::TEvConfigSubscriptionRequest::TPtr &ev, const TActorContext &ctx); void Handle(TEvConsole::TEvConfigSubscriptionCanceled::TPtr &ev, const TActorContext &ctx); From dd90e64b680d3fb14fa8439eca225f864c24fef5 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 19:54:10 +0000 Subject: [PATCH 37/54] WIP --- ydb/core/cms/console/console_configs_provider.cpp | 9 +++++++-- ydb/core/cms/console/util/config_index.h | 1 + ydb/core/protos/console_config.proto | 1 + 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ydb/core/cms/console/console_configs_provider.cpp b/ydb/core/cms/console/console_configs_provider.cpp index bd62c95667bb..94271ea719d1 100644 --- a/ydb/core/cms/console/console_configs_provider.cpp +++ b/ydb/core/cms/console/console_configs_provider.cpp @@ -1281,8 +1281,13 @@ void TConfigsProvider::UpdateConfig(TInMemorySubscription::TPtr subscription, subscription->VolatileYamlConfigHashes = VolatileYamlConfigHashes; - if (YamlConfigPerDatabase.contains(subscription->Tenant)) { - request->Record.SetDatabaseConfig(YamlConfigPerDatabase[subscription->Tenant].Config); + if (auto it = YamlConfigPerDatabase.find(subscription->Tenant); it != YamlConfigPerDatabase.end()) { + if (!subscription->DatabaseYamlConfigVersion || *subscription->DatabaseYamlConfigVersion != it->second.Version) { + subscription->DatabaseYamlConfigVersion = it->second.Version; + request->Record.SetDatabaseConfig(YamlConfigPerDatabase[subscription->Tenant].Config); + } else { + request->Record.SetDatabaseConfigNotChanged(true); + } } ctx.Send(subscription->Worker, request.Release()); diff --git a/ydb/core/cms/console/util/config_index.h b/ydb/core/cms/console/util/config_index.h index 3a6d56c336a5..71fa72ca1a6e 100644 --- a/ydb/core/cms/console/util/config_index.h +++ b/ydb/core/cms/console/util/config_index.h @@ -669,6 +669,7 @@ struct TInMemorySubscription : public TThrRefBase { ui64 YamlApiVersion = 0; ui64 YamlConfigVersion = 0; TMap VolatileYamlConfigHashes; + std::optional DatabaseYamlConfigVersion; bool FirstUpdateSent = false; diff --git a/ydb/core/protos/console_config.proto b/ydb/core/protos/console_config.proto index 46b5a95098de..c9a2fd5e9366 100644 --- a/ydb/core/protos/console_config.proto +++ b/ydb/core/protos/console_config.proto @@ -516,6 +516,7 @@ message TConfigSubscriptionNotification { optional bool YamlConfigNotChanged = 6; repeated TVolatileYamlConfig VolatileConfigs = 7; optional string DatabaseConfig = 8; + optional bool DatabaseConfigNotChanged = 9; optional NKikimrConfig.TAppConfig RawConsoleConfig = 103; // Used for correct next startup config prediction } From 59d0c34f03d967e89ca36c4052b3ac584c639d17 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 20:24:52 +0000 Subject: [PATCH 38/54] WIP --- ydb/core/protos/console_config.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/ydb/core/protos/console_config.proto b/ydb/core/protos/console_config.proto index c9a2fd5e9366..9bba68aac481 100644 --- a/ydb/core/protos/console_config.proto +++ b/ydb/core/protos/console_config.proto @@ -461,6 +461,7 @@ message TGetNodeConfigResponse { optional NKikimrConfig.TAppConfig Config = 2; optional string YamlConfig = 3; repeated TVolatileYamlConfig VolatileConfigs = 4; + optional string DatabaseYamlConfig = 5; } message TSubscriber { From b88f924eac0d90000dcaf8b0c66639a1003e118c Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 20:28:24 +0000 Subject: [PATCH 39/54] WIP --- ydb/public/lib/deprecated/kicli/configurator.cpp | 11 +++++++++++ ydb/public/lib/deprecated/kicli/kicli.h | 2 ++ 2 files changed, 13 insertions(+) diff --git a/ydb/public/lib/deprecated/kicli/configurator.cpp b/ydb/public/lib/deprecated/kicli/configurator.cpp index 0cf7c4f67eb8..9ed4422c5c3a 100644 --- a/ydb/public/lib/deprecated/kicli/configurator.cpp +++ b/ydb/public/lib/deprecated/kicli/configurator.cpp @@ -56,6 +56,17 @@ TMap TConfigurationResult::GetVolatileYamlConfigs() const return volatileConfigs; } +bool TConfigurationResult::HasDatabaseYamlConfig() const +{ + return Record().GetGetNodeConfigResponse().HasDatabaseYamlConfig(); +} + +const TString& TConfigurationResult::GetDatabaseYamlConfig() const +{ + return Record().GetGetNodeConfigResponse().GetDatabaseYamlConfig(); +} + + TNodeConfigurator::TNodeConfigurator(TKikimr& kikimr) : Kikimr(&kikimr) { diff --git a/ydb/public/lib/deprecated/kicli/kicli.h b/ydb/public/lib/deprecated/kicli/kicli.h index aa46e6694bdf..cb3dd8787a10 100644 --- a/ydb/public/lib/deprecated/kicli/kicli.h +++ b/ydb/public/lib/deprecated/kicli/kicli.h @@ -693,6 +693,8 @@ class TConfigurationResult : public TResult { bool HasYamlConfig() const; const TString& GetYamlConfig() const; TMap GetVolatileYamlConfigs() const; + bool HasDatabaseYamlConfig() const; + const TString& GetDatabaseYamlConfig() const; const NKikimrClient::TConsoleResponse &Record() const; From 281dcfe4f64cb6c397cfcfb83f66def9e8a1e5ff Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 20:51:54 +0000 Subject: [PATCH 40/54] WIP --- ydb/core/cms/console/console_configs_provider.cpp | 8 +++++++- ydb/public/lib/deprecated/kicli/configurator.cpp | 1 - 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ydb/core/cms/console/console_configs_provider.cpp b/ydb/core/cms/console/console_configs_provider.cpp index 94271ea719d1..09beb13a05e3 100644 --- a/ydb/core/cms/console/console_configs_provider.cpp +++ b/ydb/core/cms/console/console_configs_provider.cpp @@ -709,6 +709,8 @@ void TConfigsProvider::CheckSubscription(TInMemorySubscription::TPtr subscriptio subscription->VolatileYamlConfigHashes = VolatileYamlConfigHashes; + // FIXME + ctx.Send(subscription->Worker, request.Release()); subscription->FirstUpdateSent = true; @@ -1099,6 +1101,10 @@ void TConfigsProvider::Handle(TEvConsole::TEvGetNodeConfigRequest::TPtr &ev, con item.SetId(id); item.SetConfig(config); } + + if (auto it = YamlConfigPerDatabase.find(rec.GetNode().GetTenant()); it != YamlConfigPerDatabase.end()) { + response->Record.SetDatabaseYamlConfig(it->second.Config); + } } ctx.Send(ev->Sender, response.Release(), 0, ev->Cookie); @@ -1284,7 +1290,7 @@ void TConfigsProvider::UpdateConfig(TInMemorySubscription::TPtr subscription, if (auto it = YamlConfigPerDatabase.find(subscription->Tenant); it != YamlConfigPerDatabase.end()) { if (!subscription->DatabaseYamlConfigVersion || *subscription->DatabaseYamlConfigVersion != it->second.Version) { subscription->DatabaseYamlConfigVersion = it->second.Version; - request->Record.SetDatabaseConfig(YamlConfigPerDatabase[subscription->Tenant].Config); + request->Record.SetDatabaseConfig(it->second.Config); } else { request->Record.SetDatabaseConfigNotChanged(true); } diff --git a/ydb/public/lib/deprecated/kicli/configurator.cpp b/ydb/public/lib/deprecated/kicli/configurator.cpp index 9ed4422c5c3a..8555d980c0cb 100644 --- a/ydb/public/lib/deprecated/kicli/configurator.cpp +++ b/ydb/public/lib/deprecated/kicli/configurator.cpp @@ -66,7 +66,6 @@ const TString& TConfigurationResult::GetDatabaseYamlConfig() const return Record().GetGetNodeConfigResponse().GetDatabaseYamlConfig(); } - TNodeConfigurator::TNodeConfigurator(TKikimr& kikimr) : Kikimr(&kikimr) { From 5dbd0440385a9f0af505a1d5e26bceb17f83da35 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 20:53:08 +0000 Subject: [PATCH 41/54] WIP --- ydb/core/cms/console/console_configs_manager.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/ydb/core/cms/console/console_configs_manager.cpp b/ydb/core/cms/console/console_configs_manager.cpp index 6b9d0dcbda90..5451f4e1c340 100644 --- a/ydb/core/cms/console/console_configs_manager.cpp +++ b/ydb/core/cms/console/console_configs_manager.cpp @@ -821,7 +821,6 @@ void TConfigsManager::Handle(TEvConsole::TEvSetYamlConfigRequest::TPtr &ev, cons } void TConfigsManager::FailReplaceConfig(TActorId Sender, const TString& error, const TActorContext &ctx) { - // TODO: +audit auto resp = MakeHolder(); resp->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); auto *issue = resp->Record.AddIssues(); From 87e25e4b6193cb46517f9a0f7bf35b566b0e176b Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 21:12:50 +0000 Subject: [PATCH 42/54] WIP --- ydb/core/cms/console/console_configs_provider.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ydb/core/cms/console/console_configs_provider.cpp b/ydb/core/cms/console/console_configs_provider.cpp index 09beb13a05e3..603c9177fa72 100644 --- a/ydb/core/cms/console/console_configs_provider.cpp +++ b/ydb/core/cms/console/console_configs_provider.cpp @@ -709,7 +709,14 @@ void TConfigsProvider::CheckSubscription(TInMemorySubscription::TPtr subscriptio subscription->VolatileYamlConfigHashes = VolatileYamlConfigHashes; - // FIXME + if (auto it = YamlConfigPerDatabase.find(subscription->Tenant); it != YamlConfigPerDatabase.end()) { + if (!subscription->DatabaseYamlConfigVersion || *subscription->DatabaseYamlConfigVersion != it->second.Version) { + subscription->DatabaseYamlConfigVersion = it->second.Version; + request->Record.SetDatabaseConfig(it->second.Config); + } else { + request->Record.SetDatabaseConfigNotChanged(true); + } + } ctx.Send(subscription->Worker, request.Release()); From 4d3ce73f01c174445db580c3455de73dd2734f13 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 21:27:53 +0000 Subject: [PATCH 43/54] WIP --- ydb/core/cms/console/configs_dispatcher.cpp | 17 +++++++++++++---- ydb/core/config/init/init.cpp | 16 ++++++++++++---- ydb/core/config/init/init.h | 2 ++ 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/ydb/core/cms/console/configs_dispatcher.cpp b/ydb/core/cms/console/configs_dispatcher.cpp index 72f2dee849a3..e4cb4f0b08b6 100644 --- a/ydb/core/cms/console/configs_dispatcher.cpp +++ b/ydb/core/cms/console/configs_dispatcher.cpp @@ -767,25 +767,34 @@ class TConfigurationResult { public: // TODO make ref - const NKikimrConfig::TAppConfig& GetConfig() const { + const NKikimrConfig::TAppConfig& GetConfig() const override { return Config; } - bool HasYamlConfig() const { + bool HasYamlConfig() const override { return !YamlConfig.empty(); } - const TString& GetYamlConfig() const { + const TString& GetYamlConfig() const override { return YamlConfig; } - TMap GetVolatileYamlConfigs() const { + TMap GetVolatileYamlConfigs() const override { return VolatileYamlConfigs; } + bool HasDatabaseYamlConfig() const override { + return !DatabaseYamlConfig.empty(); + } + + const TString& GetDatabaseYamlConfig() const override { + return DatabaseYamlConfig; + } + NKikimrConfig::TAppConfig Config; TString YamlConfig; TMap VolatileYamlConfigs; + TString DatabaseYamlConfig; }; void TConfigsDispatcher::UpdateCandidateStartupConfig(TEvConsole::TEvConfigSubscriptionNotification::TPtr &ev) diff --git a/ydb/core/config/init/init.cpp b/ydb/core/config/init/init.cpp index 574ee1081f7c..da5574a7ea1d 100644 --- a/ydb/core/config/init/init.cpp +++ b/ydb/core/config/init/init.cpp @@ -318,21 +318,29 @@ class TDynConfigResultWrapper : Result(std::move(result)) {} - const NKikimrConfig::TAppConfig& GetConfig() const { + const NKikimrConfig::TAppConfig& GetConfig() const override { return Result.GetConfig(); } - bool HasYamlConfig() const { + bool HasYamlConfig() const override { return Result.HasYamlConfig(); } - const TString& GetYamlConfig() const { + const TString& GetYamlConfig() const override { return Result.GetYamlConfig(); } - TMap GetVolatileYamlConfigs() const { + TMap GetVolatileYamlConfigs() const override { return Result.GetVolatileYamlConfigs(); } + + bool HasDatabaseYamlConfig() const override { + return Result.HasDatabaseYamlConfig(); + } + + const TString& GetDatabaseYamlConfig() const override { + return Result.GetDatabaseYamlConfig(); + } }; class TDefaultDynConfigClient diff --git a/ydb/core/config/init/init.h b/ydb/core/config/init/init.h index 9786ee5eba85..fb10a0864120 100644 --- a/ydb/core/config/init/init.h +++ b/ydb/core/config/init/init.h @@ -159,6 +159,8 @@ class IConfigurationResult { virtual bool HasYamlConfig() const = 0; virtual const TString& GetYamlConfig() const = 0; virtual TMap GetVolatileYamlConfigs() const = 0; + virtual bool HasDatabaseYamlConfig() const = 0; + virtual const TString& GetDatabaseYamlConfig() const = 0; }; class IDynConfigClient { From e7902999d7117b3ca716d8f938614637eff13387 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 21:33:10 +0000 Subject: [PATCH 44/54] WIP --- ydb/core/cms/console/configs_dispatcher.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ydb/core/cms/console/configs_dispatcher.cpp b/ydb/core/cms/console/configs_dispatcher.cpp index e4cb4f0b08b6..d98ebf166eec 100644 --- a/ydb/core/cms/console/configs_dispatcher.cpp +++ b/ydb/core/cms/console/configs_dispatcher.cpp @@ -815,6 +815,9 @@ try { dcClient->SavedResult = configs; configs->Config = rec.GetRawConsoleConfig(); configs->YamlConfig = rec.GetYamlConfig(); + if (rec.HasDatabaseConfig()) { + configs->DatabaseYamlConfig = rec.GetDatabaseConfig(); + } // TODO volatile RecordedInitialConfiguratorDeps->DynConfigClient = std::move(dcClient); auto deps = RecordedInitialConfiguratorDeps->GetDeps(); From 962634af6506c1ad3a05084ba89819a14ed37739 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 22:22:37 +0000 Subject: [PATCH 45/54] WIP --- ydb/core/cms/console/configs_dispatcher.cpp | 2 ++ ydb/core/config/init/init.cpp | 3 ++- ydb/library/yaml_config/yaml_config.cpp | 3 +++ ydb/library/yaml_config/yaml_config.h | 1 + 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/ydb/core/cms/console/configs_dispatcher.cpp b/ydb/core/cms/console/configs_dispatcher.cpp index d98ebf166eec..356f9df55b47 100644 --- a/ydb/core/cms/console/configs_dispatcher.cpp +++ b/ydb/core/cms/console/configs_dispatcher.cpp @@ -262,6 +262,7 @@ class TConfigsDispatcher : public TActorBootstrapped { TString YamlConfig; TMap VolatileYamlConfigs; TMap VolatileYamlConfigHashes; + std::optional DatabaseYamlConfig; TString ResolvedYamlConfig; TString ResolvedJsonConfig; NKikimrConfig::TAppConfig YamlProtoConfig; @@ -378,6 +379,7 @@ NKikimrConfig::TAppConfig TConfigsDispatcher::ParseYamlProtoConfig() VolatileYamlConfigs, Labels, newYamlProtoConfig, + DatabaseYamlConfig, &ResolvedYamlConfig, &ResolvedJsonConfig); } catch (const yexception& ex) { diff --git a/ydb/core/config/init/init.cpp b/ydb/core/config/init/init.cpp index da5574a7ea1d..831ffed06d98 100644 --- a/ydb/core/config/init/init.cpp +++ b/ydb/core/config/init/init.cpp @@ -699,7 +699,8 @@ NKikimrConfig::TAppConfig GetYamlConfigFromResult(const IConfigurationResult& re result.GetYamlConfig(), result.GetVolatileYamlConfigs(), labels, - yamlConfig); + yamlConfig, + result.HasDatabaseYamlConfig() ? std::optional{result.GetDatabaseYamlConfig()} : std::nullopt); } return yamlConfig; } diff --git a/ydb/library/yaml_config/yaml_config.cpp b/ydb/library/yaml_config/yaml_config.cpp index 21c4c79a95e1..c409d1fb4d0a 100644 --- a/ydb/library/yaml_config/yaml_config.cpp +++ b/ydb/library/yaml_config/yaml_config.cpp @@ -37,9 +37,12 @@ void ResolveAndParseYamlConfig( const TMap& volatileYamlConfigs, const TMap& labels, NKikimrConfig::TAppConfig& appConfig, + std::optional databaseConfig, TString* resolvedYamlConfig, TString* resolvedJsonConfig) { + // TODO + Y_UNUSED(databaseConfig); auto tree = NFyaml::TDocument::Parse(yamlConfig); for (auto& [_, config] : volatileYamlConfigs) { diff --git a/ydb/library/yaml_config/yaml_config.h b/ydb/library/yaml_config/yaml_config.h index 76064529e581..e745f733d285 100644 --- a/ydb/library/yaml_config/yaml_config.h +++ b/ydb/library/yaml_config/yaml_config.h @@ -77,6 +77,7 @@ void ResolveAndParseYamlConfig( const TMap& volatileYamlConfigs, const TMap& labels, NKikimrConfig::TAppConfig& appConfig, + std::optional databaseConfig = std::nullopt, TString* resolvedYamlConfig = nullptr, TString* resolvedJsonConfig = nullptr); From dafd61c9b9205931fc3016a6cb6b5dbdde303dc1 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 22:31:49 +0000 Subject: [PATCH 46/54] WIP --- ydb/core/cms/console/configs_dispatcher.cpp | 22 ++++----------------- 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/ydb/core/cms/console/configs_dispatcher.cpp b/ydb/core/cms/console/configs_dispatcher.cpp index 356f9df55b47..03772b45d300 100644 --- a/ydb/core/cms/console/configs_dispatcher.cpp +++ b/ydb/core/cms/console/configs_dispatcher.cpp @@ -886,25 +886,11 @@ void TConfigsDispatcher::Handle(TEvConsole::TEvConfigSubscriptionNotification::T auto newYamlConfig = rec.GetYamlConfig(); - bool isYamlChanged = newYamlConfig != YamlConfig || rec.HasDatabaseConfig(); + bool isYamlChanged = newYamlConfig != YamlConfig; - if (rec.HasDatabaseConfig()) { - auto in = rec.GetDatabaseConfig(); - auto pos = in.find("config:"); - in = in.substr(pos); - TString newname; - newname += "- description: dbconfig\n"; - newname += " selector: \n"; - newname += " tenant: "; - newname += Labels.at("tenant"); - newname += "\n "; - for (ui64 i = 0; i < in.size(); i++) { - newname += in[i]; - if(in[i] == '\n' && i != 0) - newname += " "; - } - - newYamlConfig.append(newname); + if (rec.HasDatabaseConfig() && rec.GetDatabaseConfig() != DatabaseYamlConfig.value_or("")) { + DatabaseYamlConfig = rec.GetDatabaseConfig(); + isYamlChanged = true; } if (rec.VolatileConfigsSize() != VolatileYamlConfigs.size()) { From a84c1e949ad6d8eaecede4a3f3b56833088aa591 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 22:38:08 +0000 Subject: [PATCH 47/54] WIP --- ydb/core/cms/console/console_configs_subscriber.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/cms/console/console_configs_subscriber.cpp b/ydb/core/cms/console/console_configs_subscriber.cpp index dfe5ad9b857d..3496e62fcc50 100644 --- a/ydb/core/cms/console/console_configs_subscriber.cpp +++ b/ydb/core/cms/console/console_configs_subscriber.cpp @@ -220,7 +220,7 @@ class TConfigsSubscriber : public TActorBootstrapped { bool notChanged = true; if (ServeYaml) { - if (rec.HasDatabaseConfig()) { + if (!(rec.HasDatabaseConfigNotChanged() && rec.GetDatabaseConfigNotChanged())) { notChanged = false; } if (!(rec.HasYamlConfigNotChanged() && rec.GetYamlConfigNotChanged())) { From 4b2c081b0ddd91bf1e5a3340feb3fee1326f0a36 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Sun, 2 Feb 2025 23:23:00 +0000 Subject: [PATCH 48/54] WIP --- ydb/library/yaml_config/public/yaml_config.cpp | 17 +++++++++++++++++ ydb/library/yaml_config/public/yaml_config.h | 6 ++++++ ydb/library/yaml_config/yaml_config.cpp | 7 +++++-- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/ydb/library/yaml_config/public/yaml_config.cpp b/ydb/library/yaml_config/public/yaml_config.cpp index 4979c77de676..cb0ff559cda5 100644 --- a/ydb/library/yaml_config/public/yaml_config.cpp +++ b/ydb/library/yaml_config/public/yaml_config.cpp @@ -622,6 +622,23 @@ void AppendVolatileConfigs(NFyaml::TDocument& config, NFyaml::TNodeRef& volatile } } +void AppendDatabaseConfig(NFyaml::TDocument& config, NFyaml::TDocument& databaseConfig) { + auto configRoot = config.Root(); + if (!configRoot.Map().Has("selector_config")) { + configRoot.Map().Append(config.Buildf("selector_config"), config.Buildf("[]")); + } + + auto databaseConfigRoot = databaseConfig.Root(); + + auto selectors = configRoot.Map().at("selector_config").Sequence(); + selectors.Append(config.Buildf(R"( +description: Implicit DatabaseConfig node +selector: {} +)")); + auto node = databaseConfigRoot.Copy(config); + selectors.at(0).Map().Append(config.Buildf("config"), node); +} + ui64 GetVersion(const TString& config) { auto metadata = GetMetadata(config); return metadata.Version.value_or(0); diff --git a/ydb/library/yaml_config/public/yaml_config.h b/ydb/library/yaml_config/public/yaml_config.h index 7136cd6244e7..48426a03faf4 100644 --- a/ydb/library/yaml_config/public/yaml_config.h +++ b/ydb/library/yaml_config/public/yaml_config.h @@ -166,6 +166,12 @@ void AppendVolatileConfigs(NFyaml::TDocument& config, NFyaml::TDocument& volatil */ void AppendVolatileConfigs(NFyaml::TDocument& config, NFyaml::TNodeRef& volatileConfig); +/** + * Appends database config to the end of selectors list + * **Important**: Document should be correct DatabaseConfig + */ +void AppendDatabaseConfig(NFyaml::TDocument& config, NFyaml::TDocument& databaseConfig); + /** * Parses config version */ diff --git a/ydb/library/yaml_config/yaml_config.cpp b/ydb/library/yaml_config/yaml_config.cpp index c409d1fb4d0a..651e2091b700 100644 --- a/ydb/library/yaml_config/yaml_config.cpp +++ b/ydb/library/yaml_config/yaml_config.cpp @@ -41,10 +41,13 @@ void ResolveAndParseYamlConfig( TString* resolvedYamlConfig, TString* resolvedJsonConfig) { - // TODO - Y_UNUSED(databaseConfig); auto tree = NFyaml::TDocument::Parse(yamlConfig); + if (databaseConfig) { + auto d = NFyaml::TDocument::Parse(*databaseConfig); + NYamlConfig::AppendDatabaseConfig(tree, d); + } + for (auto& [_, config] : volatileYamlConfigs) { auto d = NFyaml::TDocument::Parse(config); NYamlConfig::AppendVolatileConfigs(tree, d); From f845ad9c84aa3ee96b3450d6cf43e84277c7c759 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Mon, 3 Feb 2025 07:06:41 +0000 Subject: [PATCH 49/54] WIP --- .../cms/console/console__replace_yaml_config.cpp | 6 +++++- ydb/core/cms/console/console_configs_manager.cpp | 15 ++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/ydb/core/cms/console/console__replace_yaml_config.cpp b/ydb/core/cms/console/console__replace_yaml_config.cpp index ed88565f8733..5115a68523e4 100644 --- a/ydb/core/cms/console/console__replace_yaml_config.cpp +++ b/ydb/core/cms/console/console__replace_yaml_config.cpp @@ -269,7 +269,11 @@ class TConfigsManager::TTxReplaceDatabaseYamlConfig try { Version = opCtx.Version; UpdatedConfig = opCtx.UpdatedConfig; - // Modify = opCtx.UpdatedConfig != Self->YamlConfig; + TString currentConfig; + if (auto it = Self->YamlConfigPerDatabase.find(opCtx.TargetDatabase); it != Self->YamlConfigPerDatabase.end()) { + currentConfig = it->second.Config; + } + Modify = opCtx.UpdatedConfig != currentConfig; if (IngressDatabase != opCtx.TargetDatabase) { WarnDatabaseBypass = true; diff --git a/ydb/core/cms/console/console_configs_manager.cpp b/ydb/core/cms/console/console_configs_manager.cpp index 5451f4e1c340..67b1baf90a77 100644 --- a/ydb/core/cms/console/console_configs_manager.cpp +++ b/ydb/core/cms/console/console_configs_manager.cpp @@ -142,16 +142,17 @@ void TConfigsManager::ReplaceDatabaseConfigMetadata(const TString &config, bool } void TConfigsManager::ValidateDatabaseConfig(TUpdateDatabaseConfigOpContext& opCtx) { - // FIXME try { - if (opCtx.UpdatedConfig != YamlConfig || YamlDropped) { - auto tree = NFyaml::TDocument::Parse(opCtx.UpdatedConfig); + TString currentConfig; + if (auto it = YamlConfigPerDatabase.find(opCtx.TargetDatabase); it != YamlConfigPerDatabase.end()) { + currentConfig = it->second.Config; + } + if (opCtx.UpdatedConfig != currentConfig) { + auto tree = NFyaml::TDocument::Parse(YamlConfig); + auto d = NFyaml::TDocument::Parse(opCtx.UpdatedConfig); + NYamlConfig::AppendDatabaseConfig(tree, d); auto resolved = NYamlConfig::ResolveAll(tree); - if (opCtx.Version != YamlVersion) { - ythrow yexception() << "Version mismatch"; - } - TSimpleSharedPtr unknownFieldsCollector = new NYamlConfig::TBasicUnknownFieldsCollector; std::vector errors; From 97d32dac3648c748d35d953d9fc45c32acb2bbf0 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Mon, 3 Feb 2025 07:31:36 +0000 Subject: [PATCH 50/54] WIP --- ydb/core/cms/console/configs_dispatcher.cpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ydb/core/cms/console/configs_dispatcher.cpp b/ydb/core/cms/console/configs_dispatcher.cpp index 03772b45d300..dba05407bae3 100644 --- a/ydb/core/cms/console/configs_dispatcher.cpp +++ b/ydb/core/cms/console/configs_dispatcher.cpp @@ -683,6 +683,26 @@ void TConfigsDispatcher::Handle(TEvInterconnect::TEvNodesInfo::TPtr &ev) } } str << "
" << Endl; + TAG(TH5) { + str << "Database Config" << Endl; + } + TAG_CLASS_STYLE(TDiv, "configs-dispatcher", "padding: 0 12px;") { + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap fold-yaml-config yaml-btn-3"}, {"title", "fold"}}) { + DIV_CLASS("yaml-sticky-btn") { } + } + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap unfold-yaml-config yaml-btn-2"}, {"title", "unfold"}}) { + DIV_CLASS("yaml-sticky-btn") { } + } + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap copy-yaml-config yaml-btn-1"}, {"title", "copy"}}) { + DIV_CLASS("yaml-sticky-btn") { } + } + DIV_CLASS("yaml-config-item") { + if (DatabaseYamlConfig) { + str << *DatabaseYamlConfig; + } + } + } + str << "
" << Endl; for (auto &[id, config] : VolatileYamlConfigs) { DIV() { TAG(TH5) { From 71bcb2a27b2c90d9600dd9fb4985958fa402faa5 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Mon, 3 Feb 2025 08:54:54 +0000 Subject: [PATCH 51/54] WIP --- .../cms/console/console__get_yaml_config.cpp | 18 ++++++++++-------- ydb/core/viewer/viewer_feature_flags.h | 4 ++-- .../api/protos/draft/ydb_dynamic_config.proto | 14 +++++++++----- .../src/client/draft/ydb_dynamic_config.cpp | 6 +++--- 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/ydb/core/cms/console/console__get_yaml_config.cpp b/ydb/core/cms/console/console__get_yaml_config.cpp index c99c36d98faa..876e393f380f 100644 --- a/ydb/core/cms/console/console__get_yaml_config.cpp +++ b/ydb/core/cms/console/console__get_yaml_config.cpp @@ -12,7 +12,7 @@ class TConfigsManager::TTxGetYamlConfig : public TTransactionBaseGet()->Record.HasDatabase() ? TMaybe{Request->Get()->Record.GetDatabase()} : TMaybe{}) + , IngressDatabase(Request->Get()->Record.HasDatabase() ? TMaybe{Request->Get()->Record.GetDatabase()} : TMaybe{}) { } @@ -20,17 +20,19 @@ class TConfigsManager::TTxGetYamlConfig : public TTransactionBase(); - if (Database) { - if (Self->YamlConfigPerDatabase.contains(*Database)) { - Response->Record.MutableResponse()->set_config(Self->YamlConfigPerDatabase[*Database].Config); + if (IngressDatabase) { + if (Self->YamlConfigPerDatabase.contains(*IngressDatabase)) { + Response->Record.MutableResponse()->add_identity()->set_database(*IngressDatabase); + Response->Record.MutableResponse()->add_identity()->set_version(Self->YamlVersion); + Response->Record.MutableResponse()->add_config(Self->YamlConfigPerDatabase[*IngressDatabase].Config); } return true; } - Response->Record.MutableResponse()->mutable_identity()->set_cluster(Self->ClusterName); - Response->Record.MutableResponse()->mutable_identity()->set_version(Self->YamlVersion); - Response->Record.MutableResponse()->set_config(Self->YamlConfig); + Response->Record.MutableResponse()->add_identity()->set_cluster(Self->ClusterName); + Response->Record.MutableResponse()->add_identity()->set_version(Self->YamlVersion); + Response->Record.MutableResponse()->add_config(Self->YamlConfig); for (auto &[id, cfg] : Self->VolatileYamlConfigs) { auto *config = Response->Record.MutableResponse()->add_volatile_configs(); @@ -53,7 +55,7 @@ class TConfigsManager::TTxGetYamlConfig : public TTransactionBase Response; - TMaybe Database; + TMaybe IngressDatabase; }; ITransaction *TConfigsManager::CreateTxGetYamlConfig(TEvConsole::TEvGetAllConfigsRequest::TPtr &ev) diff --git a/ydb/core/viewer/viewer_feature_flags.h b/ydb/core/viewer/viewer_feature_flags.h index 71000dd00cf5..37233da083d8 100644 --- a/ydb/core/viewer/viewer_feature_flags.h +++ b/ydb/core/viewer/viewer_feature_flags.h @@ -157,9 +157,9 @@ class TJsonFeatureFlags : public TViewerPipeClient { } } NKikimrConfig::TAppConfig appConfig; - if (AllConfigsResponse->Record.GetResponse().config()) { + if (AllConfigsResponse->Record.GetResponse().config_size()) { try { - NYamlConfig::ResolveAndParseYamlConfig(AllConfigsResponse->Record.GetResponse().config(), {}, {{"tenant", realDatabase}}, appConfig); + NYamlConfig::ResolveAndParseYamlConfig(AllConfigsResponse->Record.GetResponse().config(0), {}, {{"tenant", realDatabase}}, appConfig); } catch (const std::exception& e) { BLOG_ERROR("Failed to parse config for tenant " << realDatabase << ": " << e.what()); } diff --git a/ydb/public/api/protos/draft/ydb_dynamic_config.proto b/ydb/public/api/protos/draft/ydb_dynamic_config.proto index 9ba65d706d2f..bec05e95f7c8 100644 --- a/ydb/public/api/protos/draft/ydb_dynamic_config.proto +++ b/ydb/public/api/protos/draft/ydb_dynamic_config.proto @@ -9,8 +9,12 @@ import "ydb/public/api/protos/ydb_operation.proto"; message ConfigIdentity { // Current main config version uint64 version = 1; - // Cluster name (should be set on node with console tablet, unknown by default) - string cluster = 2; + oneof type { + // Cluster name (should be set on node with console tablet, unknown by default) + string cluster = 2; + // database name + string database = 3; + } } message SetConfigRequest { @@ -74,12 +78,12 @@ message GetConfigResponse { } message GetConfigResult { - // Main dynamic config with metadata in YAML format - string config = 1; + // Main/database dynamic config with metadata in YAML format + repeated string config = 1; // All volatile configs repeated VolatileConfig volatile_configs = 2; - ConfigIdentity identity = 3; + repeated ConfigIdentity identity = 3; } message VolatileConfigMetadata { diff --git a/ydb/public/sdk/cpp/src/client/draft/ydb_dynamic_config.cpp b/ydb/public/sdk/cpp/src/client/draft/ydb_dynamic_config.cpp index ddcbcaed3850..f30bf96ddf1c 100644 --- a/ydb/public/sdk/cpp/src/client/draft/ydb_dynamic_config.cpp +++ b/ydb/public/sdk/cpp/src/client/draft/ydb_dynamic_config.cpp @@ -156,9 +156,9 @@ class TDynamicConfigClient::TImpl : public TClientImplCommon volatileConfigs; if (Ydb::DynamicConfig::GetConfigResult result; any && any->UnpackTo(&result)) { - clusterName = result.identity().cluster(); - version = result.identity().version(); - config = result.config(); + clusterName = result.identity(0).cluster(); + version = result.identity(0).version(); + config = result.config(0); for (const auto& config : result.volatile_configs()) { volatileConfigs.emplace(config.id(), config.config()); } From 0c3a3c69b12413188710a9e30a54e5cd79f89478 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Mon, 3 Feb 2025 08:59:05 +0000 Subject: [PATCH 52/54] WIP --- ydb/core/cms/console/console__get_yaml_config.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ydb/core/cms/console/console__get_yaml_config.cpp b/ydb/core/cms/console/console__get_yaml_config.cpp index 876e393f380f..3320e8b391cc 100644 --- a/ydb/core/cms/console/console__get_yaml_config.cpp +++ b/ydb/core/cms/console/console__get_yaml_config.cpp @@ -23,7 +23,7 @@ class TConfigsManager::TTxGetYamlConfig : public TTransactionBaseYamlConfigPerDatabase.contains(*IngressDatabase)) { Response->Record.MutableResponse()->add_identity()->set_database(*IngressDatabase); - Response->Record.MutableResponse()->add_identity()->set_version(Self->YamlVersion); + Response->Record.MutableResponse()->add_identity()->set_version(Self->YamlConfigPerDatabase[*IngressDatabase].Version); Response->Record.MutableResponse()->add_config(Self->YamlConfigPerDatabase[*IngressDatabase].Config); } @@ -34,6 +34,12 @@ class TConfigsManager::TTxGetYamlConfig : public TTransactionBaseRecord.MutableResponse()->add_identity()->set_version(Self->YamlVersion); Response->Record.MutableResponse()->add_config(Self->YamlConfig); + for (const auto& [database, config] : Self->YamlConfigPerDatabase) { + Response->Record.MutableResponse()->add_identity()->set_database(database); + Response->Record.MutableResponse()->add_identity()->set_version(config.Version); + Response->Record.MutableResponse()->add_config(config.Config); + } + for (auto &[id, cfg] : Self->VolatileYamlConfigs) { auto *config = Response->Record.MutableResponse()->add_volatile_configs(); config->set_id(id); From 4016ea15b878469c25782afacca51d8c27b6b87d Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Mon, 3 Feb 2025 10:45:47 +0000 Subject: [PATCH 53/54] WIP --- ydb/tests/functional/audit/ya.make | 1 + 1 file changed, 1 insertion(+) diff --git a/ydb/tests/functional/audit/ya.make b/ydb/tests/functional/audit/ya.make index e843ec0fd0be..d9d6e3b94fbb 100644 --- a/ydb/tests/functional/audit/ya.make +++ b/ydb/tests/functional/audit/ya.make @@ -2,6 +2,7 @@ PY3TEST() FORK_SUBTESTS() FORK_TEST_FILES() +SPLIT_FACTOR(100) SIZE(MEDIUM) ENV(YDB_USE_IN_MEMORY_PDISKS=true) From ea9282ed8c2439f61ac2c1bbd56721935ae87861 Mon Sep 17 00:00:00 2001 From: Innokentii Mokin Date: Mon, 3 Feb 2025 10:49:32 +0000 Subject: [PATCH 54/54] WIP --- ydb/tests/functional/audit/ya.make | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ydb/tests/functional/audit/ya.make b/ydb/tests/functional/audit/ya.make index d9d6e3b94fbb..8c49ebc0af28 100644 --- a/ydb/tests/functional/audit/ya.make +++ b/ydb/tests/functional/audit/ya.make @@ -2,6 +2,10 @@ PY3TEST() FORK_SUBTESTS() FORK_TEST_FILES() +# It is necessary to run all tests +# in separate chunks because our +# audit log capture method is unreliable +# and therefore some tests may affect neighbouring ones SPLIT_FACTOR(100) SIZE(MEDIUM)