diff --git a/ydb/core/cms/console/configs_dispatcher.cpp b/ydb/core/cms/console/configs_dispatcher.cpp index 66847e9b9dcf..dba05407bae3 100644 --- a/ydb/core/cms/console/configs_dispatcher.cpp +++ b/ydb/core/cms/console/configs_dispatcher.cpp @@ -262,6 +262,7 @@ class TConfigsDispatcher : public TActorBootstrapped { TString YamlConfig; TMap VolatileYamlConfigs; TMap VolatileYamlConfigHashes; + std::optional DatabaseYamlConfig; TString ResolvedYamlConfig; TString ResolvedJsonConfig; NKikimrConfig::TAppConfig YamlProtoConfig; @@ -378,6 +379,7 @@ NKikimrConfig::TAppConfig TConfigsDispatcher::ParseYamlProtoConfig() VolatileYamlConfigs, Labels, newYamlProtoConfig, + DatabaseYamlConfig, &ResolvedYamlConfig, &ResolvedJsonConfig); } catch (const yexception& ex) { @@ -681,6 +683,26 @@ void TConfigsDispatcher::Handle(TEvInterconnect::TEvNodesInfo::TPtr &ev) } } str << "
" << Endl; + TAG(TH5) { + str << "Database Config" << Endl; + } + TAG_CLASS_STYLE(TDiv, "configs-dispatcher", "padding: 0 12px;") { + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap fold-yaml-config yaml-btn-3"}, {"title", "fold"}}) { + DIV_CLASS("yaml-sticky-btn") { } + } + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap unfold-yaml-config yaml-btn-2"}, {"title", "unfold"}}) { + DIV_CLASS("yaml-sticky-btn") { } + } + TAG_ATTRS(TDiv, {{"class", "yaml-sticky-btn-wrap copy-yaml-config yaml-btn-1"}, {"title", "copy"}}) { + DIV_CLASS("yaml-sticky-btn") { } + } + DIV_CLASS("yaml-config-item") { + if (DatabaseYamlConfig) { + str << *DatabaseYamlConfig; + } + } + } + str << "
" << Endl; for (auto &[id, config] : VolatileYamlConfigs) { DIV() { TAG(TH5) { @@ -767,25 +789,34 @@ class TConfigurationResult { public: // TODO make ref - const NKikimrConfig::TAppConfig& GetConfig() const { + const NKikimrConfig::TAppConfig& GetConfig() const override { return Config; } - bool HasYamlConfig() const { + bool HasYamlConfig() const override { return !YamlConfig.empty(); } - const TString& GetYamlConfig() const { + const TString& GetYamlConfig() const override { return YamlConfig; } - TMap GetVolatileYamlConfigs() const { + TMap GetVolatileYamlConfigs() const override { return VolatileYamlConfigs; } + bool HasDatabaseYamlConfig() const override { + return !DatabaseYamlConfig.empty(); + } + + const TString& GetDatabaseYamlConfig() const override { + return DatabaseYamlConfig; + } + NKikimrConfig::TAppConfig Config; TString YamlConfig; TMap VolatileYamlConfigs; + TString DatabaseYamlConfig; }; void TConfigsDispatcher::UpdateCandidateStartupConfig(TEvConsole::TEvConfigSubscriptionNotification::TPtr &ev) @@ -806,6 +837,9 @@ try { dcClient->SavedResult = configs; configs->Config = rec.GetRawConsoleConfig(); configs->YamlConfig = rec.GetYamlConfig(); + if (rec.HasDatabaseConfig()) { + configs->DatabaseYamlConfig = rec.GetDatabaseConfig(); + } // TODO volatile RecordedInitialConfiguratorDeps->DynConfigClient = std::move(dcClient); auto deps = RecordedInitialConfiguratorDeps->GetDeps(); @@ -870,10 +904,15 @@ void TConfigsDispatcher::Handle(TEvConsole::TEvConfigSubscriptionNotification::T CurrentConfig = rec.GetConfig(); - const auto& newYamlConfig = rec.GetYamlConfig(); + auto newYamlConfig = rec.GetYamlConfig(); bool isYamlChanged = newYamlConfig != YamlConfig; + if (rec.HasDatabaseConfig() && rec.GetDatabaseConfig() != DatabaseYamlConfig.value_or("")) { + DatabaseYamlConfig = rec.GetDatabaseConfig(); + isYamlChanged = true; + } + if (rec.VolatileConfigsSize() != VolatileYamlConfigs.size()) { isYamlChanged = true; } diff --git a/ydb/core/cms/console/console.cpp b/ydb/core/cms/console/console.cpp index 6c856ea785e5..64e39beabbaf 100644 --- a/ydb/core/cms/console/console.cpp +++ b/ydb/core/cms/console/console.cpp @@ -186,6 +186,15 @@ void TConsole::Handle(TEvConsole::TEvSetConfigRequest::TPtr &ev, const TActorCon TxProcessor->ProcessTx(CreateTxSetConfig(ev), ctx); } +bool TConsole::HasTenant(const TString& path) const +{ + if (!TenantsManager) { + return false; + } + + return TenantsManager->HasTenant(path); +} + IActor *CreateConsole(const TActorId &tablet, TTabletStorageInfo *info) { return new TConsole(tablet, info); diff --git a/ydb/core/cms/console/console.h b/ydb/core/cms/console/console.h index 1cfc641a372e..971efc153393 100644 --- a/ydb/core/cms/console/console.h +++ b/ydb/core/cms/console/console.h @@ -375,6 +375,35 @@ namespace TEvConsole { } } } + + TEvConfigSubscriptionNotification( + ui64 generation, + const NKikimrConfig::TAppConfig &config, + const THashSet &affectedKinds, + const TString &yamlConfig, + const TMap &volatileYamlConfigs, + const NKikimrConfig::TAppConfig &rawConfig, + const TMaybe dbc) + { + Record.SetGeneration(generation); + Record.MutableConfig()->CopyFrom(config); + Record.MutableRawConsoleConfig()->CopyFrom(rawConfig); + for (ui32 kind : affectedKinds) + Record.AddAffectedKinds(kind); + + if (!yamlConfig.empty()) { + Record.SetYamlConfig(yamlConfig); + for (auto &[id, config] : volatileYamlConfigs) { + auto *volatileConfig = Record.AddVolatileConfigs(); + volatileConfig->SetId(id); + volatileConfig->SetConfig(config); + } + } + if (dbc) { + Record.SetDatabaseConfig(*dbc); + } + } + }; /** diff --git a/ydb/core/cms/console/console__get_yaml_config.cpp b/ydb/core/cms/console/console__get_yaml_config.cpp index d25d264c4878..3320e8b391cc 100644 --- a/ydb/core/cms/console/console__get_yaml_config.cpp +++ b/ydb/core/cms/console/console__get_yaml_config.cpp @@ -12,6 +12,7 @@ class TConfigsManager::TTxGetYamlConfig : public TTransactionBaseGet()->Record.HasDatabase() ? TMaybe{Request->Get()->Record.GetDatabase()} : TMaybe{}) { } @@ -19,9 +20,25 @@ class TConfigsManager::TTxGetYamlConfig : public TTransactionBase(); - Response->Record.MutableResponse()->mutable_identity()->set_cluster(Self->ClusterName); - Response->Record.MutableResponse()->mutable_identity()->set_version(Self->YamlVersion); - Response->Record.MutableResponse()->set_config(Self->YamlConfig); + if (IngressDatabase) { + if (Self->YamlConfigPerDatabase.contains(*IngressDatabase)) { + Response->Record.MutableResponse()->add_identity()->set_database(*IngressDatabase); + Response->Record.MutableResponse()->add_identity()->set_version(Self->YamlConfigPerDatabase[*IngressDatabase].Version); + Response->Record.MutableResponse()->add_config(Self->YamlConfigPerDatabase[*IngressDatabase].Config); + } + + return true; + } + + Response->Record.MutableResponse()->add_identity()->set_cluster(Self->ClusterName); + Response->Record.MutableResponse()->add_identity()->set_version(Self->YamlVersion); + Response->Record.MutableResponse()->add_config(Self->YamlConfig); + + for (const auto& [database, config] : Self->YamlConfigPerDatabase) { + Response->Record.MutableResponse()->add_identity()->set_database(database); + Response->Record.MutableResponse()->add_identity()->set_version(config.Version); + Response->Record.MutableResponse()->add_config(config.Config); + } for (auto &[id, cfg] : Self->VolatileYamlConfigs) { auto *config = Response->Record.MutableResponse()->add_volatile_configs(); @@ -44,6 +61,7 @@ class TConfigsManager::TTxGetYamlConfig : public TTransactionBase Response; + TMaybe IngressDatabase; }; ITransaction *TConfigsManager::CreateTxGetYamlConfig(TEvConsole::TEvGetAllConfigsRequest::TPtr &ev) diff --git a/ydb/core/cms/console/console__replace_yaml_config.cpp b/ydb/core/cms/console/console__replace_yaml_config.cpp index 6eeb03ea5e46..5115a68523e4 100644 --- a/ydb/core/cms/console/console__replace_yaml_config.cpp +++ b/ydb/core/cms/console/console__replace_yaml_config.cpp @@ -7,40 +7,207 @@ #include #include #include +#include namespace NKikimr::NConsole { using namespace NKikimrConsole; -class TConfigsManager::TTxReplaceYamlConfig : public TTransactionBase { +class TConfigsManager::TTxReplaceYamlConfigBase + : public TTransactionBase +{ template - TTxReplaceYamlConfig(TConfigsManager *self, - T &ev, - bool force) - : TBase(self) - , Config(ev->Get()->Record.GetRequest().config()) - , Peer(ev->Get()->Record.GetPeerName()) - , Sender(ev->Sender) - , UserToken(ev->Get()->Record.GetUserToken()) - , Force(force) - , AllowUnknownFields(ev->Get()->Record.GetRequest().allow_unknown_fields()) - , DryRun(ev->Get()->Record.GetRequest().dry_run()) + TTxReplaceYamlConfigBase( + TConfigsManager *self, + T &ev, + bool force) + : TBase(self) + , Config(ev->Get()->Record.GetRequest().config()) + , Peer(ev->Get()->Record.GetPeerName()) + , Sender(ev->Sender) + , UserToken(ev->Get()->Record.GetUserToken()) + , Force(force) + , AllowUnknownFields(ev->Get()->Record.GetRequest().allow_unknown_fields()) + , DryRun(ev->Get()->Record.GetRequest().dry_run()) + , IngressDatabase(ev->Get()->Record.HasDatabase() ? TMaybe{ev->Get()->Record.GetDatabase()} : TMaybe{}) { } public: - TTxReplaceYamlConfig(TConfigsManager *self, - TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev) - : TTxReplaceYamlConfig(self, ev, false) + TTxReplaceYamlConfigBase( + TConfigsManager *self, + TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev) + : TTxReplaceYamlConfigBase(self, ev, false) { } - TTxReplaceYamlConfig(TConfigsManager *self, - TEvConsole::TEvSetYamlConfigRequest::TPtr &ev) - : TTxReplaceYamlConfig(self, ev, true) + TTxReplaceYamlConfigBase( + TConfigsManager *self, + TEvConsole::TEvSetYamlConfigRequest::TPtr &ev) + : TTxReplaceYamlConfigBase(self, ev, true) { } + THolder FillResponse(const TUpdateConfigOpBaseContext& opCtx, auto& ev, auto errorLevel, const TActorContext &ctx) { + for (auto& [path, info] : opCtx.UnknownFields) { + auto *issue = ev->Record.AddIssues(); + issue->set_severity(errorLevel); + issue->set_message(TStringBuilder{} << "Unknown key# " << info.first << " in proto# " << info.second << " found in path# " << path); + } + + for (auto& [path, info] : opCtx.DeprecatedFields) { + auto *issue = ev->Record.AddIssues(); + issue->set_severity(NYql::TSeverityIds::S_WARNING); + issue->set_message(TStringBuilder{} << "Deprecated key# " << info.first << " in proto# " << info.second << " found in path# " << path); + } + + return MakeHolder(Sender, ctx.SelfID, ev.Release()); + } + + void HandleError(const TString& error, const TActorContext& ctx) { + Error = true; + auto ev = MakeHolder(); + ev->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); + auto *issue = ev->Record.AddIssues(); + issue->set_severity(NYql::TSeverityIds::S_ERROR); + issue->set_message(error); + ErrorReason = error; + Response = MakeHolder(Sender, ctx.SelfID, ev.Release()); + } + +protected: + const TString Config; + const TString Peer; + const TActorId Sender; + const NACLib::TUserToken UserToken; + const bool Force = false; + const bool AllowUnknownFields = false; + const bool DryRun = false; + THolder Response; + bool Error = false; + TString ErrorReason; + bool Modify = false; + TSimpleSharedPtr UnknownFieldsCollector = nullptr; + TMaybe IngressDatabase; + bool WarnDatabaseBypass = false; +}; + +class TConfigsManager::TTxReplaceMainYamlConfig + : public TConfigsManager::TTxReplaceYamlConfigBase +{ +public: + TTxReplaceMainYamlConfig( + TConfigsManager *self, + TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev) + : TTxReplaceYamlConfigBase(self, ev) + { + } + + TTxReplaceMainYamlConfig( + TConfigsManager *self, + TEvConsole::TEvSetYamlConfigRequest::TPtr &ev) + : TTxReplaceYamlConfigBase(self, ev) + { + } + + bool Execute(TTransactionContext &txc, const TActorContext &ctx) override + { + NIceDb::TNiceDb db(txc.DB); + + TUpdateConfigOpContext opCtx; + Self->ReplaceMainConfigMetadata(Config, false, opCtx); + Self->ValidateMainConfig(opCtx); + + bool hasForbiddenUnknown = !opCtx.UnknownFields.empty() && !AllowUnknownFields; + if (opCtx.Error) { + HandleError(opCtx.Error.value(), ctx); + return true; + } + + try { + Version = opCtx.Version; + UpdatedConfig = opCtx.UpdatedConfig; + Cluster = opCtx.Cluster; + Modify = opCtx.UpdatedConfig != Self->YamlConfig || Self->YamlDropped; + + if (IngressDatabase) { + WarnDatabaseBypass = true; + } + + if (!DryRun && !hasForbiddenUnknown) { + DoInternalAudit(txc, ctx); + + db.Table().Key(Version + 1) + .Update(UpdatedConfig) + // set config dropped by default to support rollback to previous versions + // where new config layout is not supported + // it will lead to ignoring config from new versions + .Update(true); + + /* Later we shift this boundary to support rollback and history */ + db.Table().Key(Version) + .Delete(); + } + + if (hasForbiddenUnknown) { + Error = true; + auto ev = MakeHolder(); + ev->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); + ErrorReason = "Unknown keys in config."; + Response = FillResponse(opCtx, ev, NYql::TSeverityIds::S_ERROR, ctx); + } else if (!Force) { + auto ev = MakeHolder(); + Response = FillResponse(opCtx, ev, NYql::TSeverityIds::S_WARNING, ctx); + } else { + auto ev = MakeHolder(); + Response = FillResponse(opCtx, ev, NYql::TSeverityIds::S_WARNING, ctx); + } + } + catch (const yexception& ex) { + HandleError(ex.what(), ctx); + } + + return true; + } + + void Complete(const TActorContext &ctx) override + { + LOG_DEBUG(ctx, NKikimrServices::CMS_CONFIGS, "TTxReplaceYamlConfig Complete"); + + ctx.Send(Response.Release()); + + if (!Error && Modify && !DryRun) { + AuditLogReplaceConfigTransaction( + /* peer = */ Peer, + /* userSID = */ UserToken.GetUserSID(), + /* sanitizedToken = */ UserToken.GetSanitizedToken(), + /* oldConfig = */ Self->YamlConfig, + /* newConfig = */ Config, + /* reason = */ {}, + /* success = */ true); + + Self->YamlVersion = Version + 1; + Self->YamlConfig = UpdatedConfig; + Self->YamlDropped = false; + + Self->VolatileYamlConfigs.clear(); + + auto resp = MakeHolder(Self->YamlConfig, Self->YamlConfigPerDatabase); + ctx.Send(Self->ConfigsProvider, resp.Release()); + } else if (Error && !DryRun) { + AuditLogReplaceConfigTransaction( + /* peer = */ Peer, + /* userSID = */ UserToken.GetUserSID(), + /* sanitizedToken = */ UserToken.GetSanitizedToken(), + /* oldConfig = */ Self->YamlConfig, + /* newConfig = */ Config, + /* reason = */ ErrorReason, + /* success = */ false); + } + + Self->TxProcessor->TxCompleted(this, ctx); + } + void DoInternalAudit(TTransactionContext &txc, const TActorContext &ctx) { auto logData = NKikimrConsole::TLogRecordData{}; @@ -61,104 +228,166 @@ class TConfigsManager::TTxReplaceYamlConfig : public TTransactionBaseLogger.DbLogData(UserToken.GetUserSID(), logData, txc, ctx); } +private: + ui32 Version; + TString Cluster; + TString UpdatedConfig; +}; + +class TConfigsManager::TTxReplaceDatabaseYamlConfig + : public TConfigsManager::TTxReplaceYamlConfigBase +{ +public: + TTxReplaceDatabaseYamlConfig( + TConfigsManager *self, + TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev) + : TTxReplaceYamlConfigBase(self, ev) + { + } + + TTxReplaceDatabaseYamlConfig( + TConfigsManager *self, + TEvConsole::TEvSetYamlConfigRequest::TPtr &ev) + : TTxReplaceYamlConfigBase(self, ev) + { + } + bool Execute(TTransactionContext &txc, const TActorContext &ctx) override { NIceDb::TNiceDb db(txc.DB); - TValidateConfigResult result = Self->ValidateConfigAndReplaceMetadata(Config, Force, AllowUnknownFields); - if (result.ErrorReason) { - HandleError(result.ErrorReason.value(), ctx); + + TUpdateDatabaseConfigOpContext opCtx; + Self->ReplaceDatabaseConfigMetadata(Config, false, opCtx); + Self->ValidateDatabaseConfig(opCtx); + + bool hasForbiddenUnknown = !opCtx.UnknownFields.empty() && !AllowUnknownFields; + if (opCtx.Error) { + HandleError(opCtx.Error.value(), ctx); return true; } - try { - Version = result.Version; - UpdatedConfig = result.UpdatedConfig; - Cluster = result.Cluster; - Modify = result.Modify; - - if (result.ValidationFinished) { - if (!DryRun && !result.HasForbiddenUnknown) { - DoInternalAudit(txc, ctx); - - db.Table().Key(Version + 1) - .Update(UpdatedConfig) - // set config dropped by default to support rollback to previous versions - // where new config layout is not supported - // it will lead to ignoring config from new versions - .Update(true); - - /* Later we shift this boundary to support rollback and history */ - db.Table().Key(Version) - .Delete(); - } + try { + Version = opCtx.Version; + UpdatedConfig = opCtx.UpdatedConfig; + TString currentConfig; + if (auto it = Self->YamlConfigPerDatabase.find(opCtx.TargetDatabase); it != Self->YamlConfigPerDatabase.end()) { + currentConfig = it->second.Config; } + Modify = opCtx.UpdatedConfig != currentConfig; - auto fillResponse = [&](auto& ev, auto errorLevel){ - for (auto& [path, info] : result.UnknownFields) { - auto *issue = ev->Record.AddIssues(); - issue->set_severity(errorLevel); - issue->set_message(TStringBuilder{} << "Unknown key# " << info.first << " in proto# " << info.second << " found in path# " << path); - } + if (IngressDatabase != opCtx.TargetDatabase) { + WarnDatabaseBypass = true; + } - for (auto& [path, info] : result.DeprecatedFields) { - auto *issue = ev->Record.AddIssues(); - issue->set_severity(NYql::TSeverityIds::S_WARNING); - issue->set_message(TStringBuilder{} << "Deprecated key# " << info.first << " in proto# " << info.second << " found in path# " << path); - } + if (!AppData(ctx)->FeatureFlags.GetPerDatabaseConfigAllowed()) { + Error = true; + auto ev = MakeHolder(); + auto *issue = ev->Record.AddIssues(); + ErrorReason = "Per database config is disabled"; + issue->set_severity(NYql::TSeverityIds::S_ERROR); + issue->set_message(ErrorReason); + ev->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); Response = MakeHolder(Sender, ctx.SelfID, ev.Release()); - }; + return true; + } + + if (!DryRun && !hasForbiddenUnknown) { + DoInternalAudit(txc, ctx); - if (result.HasForbiddenUnknown) { + db.Table().Key(TargetDatabase, Version + 1) + .Update(Config); + + /* Later we shift this boundary to support rollback and history */ + db.Table().Key(TargetDatabase, Version) + .Delete(); + } + + if (hasForbiddenUnknown) { Error = true; auto ev = MakeHolder(); ev->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); ErrorReason = "Unknown keys in config."; - fillResponse(ev, NYql::TSeverityIds::S_ERROR); + Response = FillResponse(opCtx, ev, NYql::TSeverityIds::S_ERROR, ctx); } else if (!Force) { auto ev = MakeHolder(); - fillResponse(ev, NYql::TSeverityIds::S_WARNING); + Response = FillResponse(opCtx, ev, NYql::TSeverityIds::S_WARNING, ctx); } else { auto ev = MakeHolder(); - fillResponse(ev, NYql::TSeverityIds::S_WARNING); + Response = FillResponse(opCtx, ev, NYql::TSeverityIds::S_WARNING, ctx); } } catch (const yexception& ex) { HandleError(ex.what(), ctx); } + return true; } + void DoInternalAudit(TTransactionContext &txc, const TActorContext &ctx) + { + TString oldConfig; + + if (Self->YamlConfigPerDatabase.contains(TargetDatabase)) { + oldConfig = Self->YamlConfigPerDatabase[TargetDatabase].Config; + } + + auto logData = NKikimrConsole::TLogRecordData{}; + + // for backward compatibility in ui + logData.MutableAction()->AddActions()->MutableModifyConfigItem()->MutableConfigItem(); + logData.AddAffectedKinds(NKikimrConsole::TConfigItem::DatabaseYamlConfigChangeItem); + + auto& databaseConfigChange = *logData.MutableDatabaseConfigChange(); + databaseConfigChange.SetDatabase(TargetDatabase); + databaseConfigChange.SetOldYamlConfig(oldConfig); + databaseConfigChange.SetNewYamlConfig(UpdatedConfig); + + Self->Logger.DbLogData(UserToken.GetUserSID(), logData, txc, ctx); + } + void Complete(const TActorContext &ctx) override { LOG_DEBUG(ctx, NKikimrServices::CMS_CONFIGS, "TTxReplaceYamlConfig Complete"); ctx.Send(Response.Release()); + TString oldConfig; + + if (Self->YamlConfigPerDatabase.contains(TargetDatabase)) { + oldConfig = Self->YamlConfigPerDatabase[TargetDatabase].Config; + } + if (!Error && Modify && !DryRun) { - AuditLogReplaceConfigTransaction( + AuditLogReplaceDatabaseConfigTransaction( /* peer = */ Peer, /* userSID = */ UserToken.GetUserSID(), /* sanitizedToken = */ UserToken.GetSanitizedToken(), - /* oldConfig = */ Self->YamlConfig, + /* database = */ TargetDatabase, + /* oldConfig = */ oldConfig, /* newConfig = */ Config, /* reason = */ {}, /* success = */ true); - Self->YamlVersion = Version + 1; - Self->YamlConfig = UpdatedConfig; - Self->YamlDropped = false; + Self->YamlConfigPerDatabase[*IngressDatabase] = TDatabaseYamlConfig { + .Config = UpdatedConfig, + .Version = Version + 1, + }; - Self->VolatileYamlConfigs.clear(); + auto resp = MakeHolder( + Self->YamlConfig, + Self->YamlConfigPerDatabase, + Self->VolatileYamlConfigs, + TargetDatabase); - auto resp = MakeHolder(Self->YamlConfig); ctx.Send(Self->ConfigsProvider, resp.Release()); } else if (Error && !DryRun) { - AuditLogReplaceConfigTransaction( + AuditLogReplaceDatabaseConfigTransaction( /* peer = */ Peer, /* userSID = */ UserToken.GetUserSID(), /* sanitizedToken = */ UserToken.GetSanitizedToken(), - /* oldConfig = */ Self->YamlConfig, + /* database = */ TargetDatabase, + /* oldConfig = */ oldConfig, /* newConfig = */ Config, /* reason = */ ErrorReason, /* success = */ false); @@ -167,43 +396,30 @@ class TConfigsManager::TTxReplaceYamlConfig : public TTransactionBaseTxProcessor->TxCompleted(this, ctx); } - void HandleError(const TString& error, const TActorContext& ctx) { - Error = true; - auto ev = MakeHolder(); - ev->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); - auto *issue = ev->Record.AddIssues(); - issue->set_severity(NYql::TSeverityIds::S_ERROR); - issue->set_message(error); - ErrorReason = error; - Response = MakeHolder(Sender, ctx.SelfID, ev.Release()); - } - private: - const TString Config; - const TString Peer; - const TActorId Sender; - const NACLib::TUserToken UserToken; - const bool Force = false; - const bool AllowUnknownFields = false; - const bool DryRun = false; - THolder Response; - bool Error = false; - TString ErrorReason; - bool Modify = false; - TSimpleSharedPtr UnknownFieldsCollector = nullptr; ui32 Version; - TString Cluster; + TString TargetDatabase; TString UpdatedConfig; }; -ITransaction *TConfigsManager::CreateTxReplaceYamlConfig(TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev) +ITransaction *TConfigsManager::CreateTxReplaceMainYamlConfig(TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev) +{ + return new TTxReplaceMainYamlConfig(this, ev); +} + +ITransaction *TConfigsManager::CreateTxSetMainYamlConfig(TEvConsole::TEvSetYamlConfigRequest::TPtr &ev) +{ + return new TTxReplaceMainYamlConfig(this, ev); +} + +ITransaction *TConfigsManager::CreateTxReplaceDatabaseYamlConfig(TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev) { - return new TTxReplaceYamlConfig(this, ev); + return new TTxReplaceDatabaseYamlConfig(this, ev); } -ITransaction *TConfigsManager::CreateTxSetYamlConfig(TEvConsole::TEvSetYamlConfigRequest::TPtr &ev) +ITransaction *TConfigsManager::CreateTxSetDatabaseYamlConfig(TEvConsole::TEvSetYamlConfigRequest::TPtr &ev) { - return new TTxReplaceYamlConfig(this, ev); + return new TTxReplaceDatabaseYamlConfig(this, ev); } } // namespace NKikimr::NConsole diff --git a/ydb/core/cms/console/console__scheme.h b/ydb/core/cms/console/console__scheme.h index b0e40979a994..8c7a0f355983 100644 --- a/ydb/core/cms/console/console__scheme.h +++ b/ydb/core/cms/console/console__scheme.h @@ -166,9 +166,19 @@ struct Schema : NIceDb::Schema { using TColumns = TableColumns; }; + struct PerTenantYamlConfig : Table<104> { + struct Path : Column<1, NScheme::NTypeIds::Utf8> {}; + struct Version : Column<2, NScheme::NTypeIds::Uint64> {}; + struct Config : Column<3, NScheme::NTypeIds::String> {}; + + + using TKey = TableKey; + using TColumns = TableColumns; + }; + using TTables = SchemaTables; + YamlConfig, PerTenantYamlConfig>; using TSettings = SchemaSettings, ExecutorLogFlushPeriod>; }; diff --git a/ydb/core/cms/console/console_audit.cpp b/ydb/core/cms/console/console_audit.cpp index 68a70bb62799..e2b6d8e8b89d 100644 --- a/ydb/core/cms/console/console_audit.cpp +++ b/ydb/core/cms/console/console_audit.cpp @@ -5,6 +5,9 @@ namespace NKikimr::NConsole { +static const TString COMPONENT_NAME = "console"; +static const TString EMPTY_VALUE = "{none}"; + void AuditLogReplaceConfigTransaction( const TString& peer, const TString& userSID, @@ -14,10 +17,31 @@ void AuditLogReplaceConfigTransaction( const TString& reason, bool success) { - static const TString COMPONENT_NAME = "console"; + auto peerName = NKikimr::NAddressClassifier::ExtractAddress(peer); - static const TString EMPTY_VALUE = "{none}"; + AUDIT_LOG( + AUDIT_PART("component", COMPONENT_NAME) + AUDIT_PART("remote_address", (!peerName.empty() ? peerName : EMPTY_VALUE)) + AUDIT_PART("subject", (!userSID.empty() ? userSID : EMPTY_VALUE)) + AUDIT_PART("sanitized_token", (!sanitizedToken.empty() ? sanitizedToken : EMPTY_VALUE)) + AUDIT_PART("status", TString(success ? "SUCCESS" : "ERROR")) + AUDIT_PART("reason", reason, !reason.empty()) + AUDIT_PART("operation", TString("REPLACE DYNCONFIG")) + AUDIT_PART("old_config", oldConfig) + AUDIT_PART("new_config", newConfig) + ); +} +void AuditLogReplaceDatabaseConfigTransaction( + const TString& peer, + const TString& userSID, + const TString& sanitizedToken, + const TString& database, + const TString& oldConfig, + const TString& newConfig, + const TString& reason, + bool success) +{ auto peerName = NKikimr::NAddressClassifier::ExtractAddress(peer); AUDIT_LOG( @@ -25,9 +49,10 @@ void AuditLogReplaceConfigTransaction( AUDIT_PART("remote_address", (!peerName.empty() ? peerName : EMPTY_VALUE)) AUDIT_PART("subject", (!userSID.empty() ? userSID : EMPTY_VALUE)) AUDIT_PART("sanitized_token", (!sanitizedToken.empty() ? sanitizedToken : EMPTY_VALUE)) + AUDIT_PART("database", (!database.empty() ? database : EMPTY_VALUE)) AUDIT_PART("status", TString(success ? "SUCCESS" : "ERROR")) AUDIT_PART("reason", reason, !reason.empty()) - AUDIT_PART("operation", TString("REPLACE DYNCONFIG")) + AUDIT_PART("operation", TString("REPLACE DATABASE CONFIG")) AUDIT_PART("old_config", oldConfig) AUDIT_PART("new_config", newConfig) ); diff --git a/ydb/core/cms/console/console_audit.h b/ydb/core/cms/console/console_audit.h index b3c94e37e1fc..77a55db7837f 100644 --- a/ydb/core/cms/console/console_audit.h +++ b/ydb/core/cms/console/console_audit.h @@ -13,4 +13,14 @@ void AuditLogReplaceConfigTransaction( const TString& reason, bool success); +void AuditLogReplaceDatabaseConfigTransaction( + const TString& peer, + const TString& userSID, + const TString& sanitizedToken, + const TString& database, + const TString& oldConfig, + const TString& newConfig, + const TString& reason, + bool success); + } // namespace NKikimr::NConsole diff --git a/ydb/core/cms/console/console_configs_manager.cpp b/ydb/core/cms/console/console_configs_manager.cpp index bd572718568b..67b1baf90a77 100644 --- a/ydb/core/cms/console/console_configs_manager.cpp +++ b/ydb/core/cms/console/console_configs_manager.cpp @@ -52,35 +52,37 @@ bool TConfigsManager::CheckConfig(const NKikimrConsole::TConfigsConfig &config, return true; } -TConfigsManager::TValidateConfigResult TConfigsManager::ValidateConfigAndReplaceMetadata(const TString &config, bool force, bool allowUnknownFields) { - TValidateConfigResult result; +void TConfigsManager::ReplaceMainConfigMetadata(const TString &config, bool force, TUpdateConfigOpContext& opCtx) { try { if (!force) { auto metadata = NYamlConfig::GetMetadata(config); - result.Cluster = metadata.Cluster.value_or(TString("unknown")); - result.Version = metadata.Version.value_or(0); + opCtx.Cluster = metadata.Cluster.value_or(TString("unknown")); + opCtx.Version = metadata.Version.value_or(0); } else { - result.Cluster = ClusterName; - result.Version = YamlVersion; + opCtx.Cluster = ClusterName; + opCtx.Version = YamlVersion; } - result.UpdatedConfig = NYamlConfig::ReplaceMetadata(config, NYamlConfig::TMetadata{ - .Version = result.Version + 1, - .Cluster = result.Cluster, + opCtx.UpdatedConfig = NYamlConfig::ReplaceMetadata(config, NYamlConfig::TMainMetadata{ + .Version = opCtx.Version + 1, + .Cluster = opCtx.Cluster, }); + } catch (const yexception &e) { + opCtx.Error = e.what(); + } +} - result.HasForbiddenUnknown = false; - if (result.UpdatedConfig != YamlConfig || YamlDropped) { - result.Modify = true; - - auto tree = NFyaml::TDocument::Parse(result.UpdatedConfig); +void TConfigsManager::ValidateMainConfig(TUpdateConfigOpContext& opCtx) { + try { + if (opCtx.UpdatedConfig != YamlConfig || YamlDropped) { + auto tree = NFyaml::TDocument::Parse(opCtx.UpdatedConfig); auto resolved = NYamlConfig::ResolveAll(tree); - if (ClusterName != result.Cluster) { + if (ClusterName != opCtx.Cluster) { ythrow yexception() << "ClusterName mismatch"; } - if (result.Version != YamlVersion) { + if (opCtx.Version != YamlVersion) { ythrow yexception() << "Version mismatch"; } @@ -103,19 +105,82 @@ TConfigsManager::TValidateConfigResult TConfigsManager::ValidateConfigAndReplace for (const auto& [path, info] : unknownFieldsCollector->GetUnknownKeys()) { if (deprecatedPaths.contains(path)) { - result.DeprecatedFields[path] = info; + opCtx.DeprecatedFields[path] = info; } else { - result.UnknownFields[path] = info; + opCtx.UnknownFields[path] = info; } } + } + } catch (const yexception &e) { + opCtx.Error = e.what(); + } +} + +void TConfigsManager::ReplaceDatabaseConfigMetadata(const TString &config, bool force, TUpdateDatabaseConfigOpContext& opCtx) { + try { + auto metadata = NYamlConfig::GetDatabaseMetadata(config); + + if (!metadata.Database) { + ythrow yexception() << "metadata.database is not present, unable to infer target database"; + } + + opCtx.TargetDatabase = *metadata.Database; - result.HasForbiddenUnknown = !result.UnknownFields.empty() && !allowUnknownFields; - result.ValidationFinished = true; + if (!force) { + opCtx.Version = metadata.Version.value_or(0); + } else { + opCtx.Version = YamlVersion; } + + opCtx.UpdatedConfig = NYamlConfig::ReplaceMetadata(config, NYamlConfig::TDatabaseMetadata{ + .Version = opCtx.Version + 1, + .Database = opCtx.TargetDatabase, + }); } catch (const yexception &e) { - result.ErrorReason = e.what(); + opCtx.Error = e.what(); + } +} + +void TConfigsManager::ValidateDatabaseConfig(TUpdateDatabaseConfigOpContext& opCtx) { + try { + TString currentConfig; + if (auto it = YamlConfigPerDatabase.find(opCtx.TargetDatabase); it != YamlConfigPerDatabase.end()) { + currentConfig = it->second.Config; + } + if (opCtx.UpdatedConfig != currentConfig) { + auto tree = NFyaml::TDocument::Parse(YamlConfig); + auto d = NFyaml::TDocument::Parse(opCtx.UpdatedConfig); + NYamlConfig::AppendDatabaseConfig(tree, d); + auto resolved = NYamlConfig::ResolveAll(tree); + + TSimpleSharedPtr unknownFieldsCollector = new NYamlConfig::TBasicUnknownFieldsCollector; + + std::vector errors; + for (auto& [_, config] : resolved.Configs) { + auto cfg = NYamlConfig::YamlToProto( + config.second, + true, + true, + unknownFieldsCollector); + NKikimr::NConfig::EValidationResult result = NKikimr::NConfig::ValidateConfig(cfg, errors); + if (result == NKikimr::NConfig::EValidationResult::Error) { + ythrow yexception() << errors.front(); + } + } + + const auto& deprecatedPaths = NKikimrConfig::TAppConfig::GetReservedChildrenPaths(); + + for (const auto& [path, info] : unknownFieldsCollector->GetUnknownKeys()) { + if (deprecatedPaths.contains(path)) { + opCtx.DeprecatedFields[path] = info; + } else { + opCtx.UnknownFields[path] = info; + } + } + } + } catch (const yexception &e) { + opCtx.Error = e.what(); } - return result; } void TConfigsManager::Bootstrap(const TActorContext &ctx) @@ -391,14 +456,18 @@ bool TConfigsManager::DbLoadState(TTransactionContext &txc, auto subscriptionRowset = db.Table().Range().Select(); auto validatorsRowset = db.Table().Range().Select(); auto yamlConfigRowset = db.Table().Reverse().Select(); + auto perDatabaseYamlConfigRowset = db.Table().Select(); if (!configItemRowset.IsReady() || !nextConfigItemIdRow.IsReady() || !nextSubscriptionIdRow.IsReady() || !subscriptionRowset.IsReady() || !validatorsRowset.IsReady() - || !yamlConfigRowset.IsReady()) + || !yamlConfigRowset.IsReady() + || !perDatabaseYamlConfigRowset.IsReady()) + { return false; + } if (nextConfigItemIdRow.IsValid()) { TString value = nextConfigItemIdRow.GetValue(); @@ -432,6 +501,21 @@ bool TConfigsManager::DbLoadState(TTransactionContext &txc, YamlDropped = false; } + while (!perDatabaseYamlConfigRowset.EndOfSet()) { + TString tenant = perDatabaseYamlConfigRowset.GetValue(); + ui32 version = perDatabaseYamlConfigRowset.GetValue(); + TString config = perDatabaseYamlConfigRowset.GetValue(); + + YamlConfigPerDatabase[tenant] = TDatabaseYamlConfig { + .Config = config, + .Version = version, + }; + + if (!perDatabaseYamlConfigRowset.Next()) { + return false; + } + } + while (!configItemRowset.EndOfSet()) { ui64 id = configItemRowset.GetValue(); ui64 generation = configItemRowset.GetValue(); @@ -683,12 +767,68 @@ void TConfigsManager::Handle(TEvConsole::TEvToggleConfigValidatorRequest::TPtr & void TConfigsManager::Handle(TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev, const TActorContext &ctx) { - TxProcessor->ProcessTx(CreateTxReplaceYamlConfig(ev), ctx); + auto metadata = NYamlConfig::GetGenericMetadata(ev->Get()->Record.GetRequest().config()); + + std::visit(TOverloaded{ + [&](const NYamlConfig::TMainMetadata& /* value */) { + TxProcessor->ProcessTx(CreateTxReplaceMainYamlConfig(ev), ctx); + }, + [&](const NYamlConfig::TDatabaseMetadata& value) { + if (!value.Database || !Self.HasTenant(*value.Database)) { + return FailReplaceConfig(ev->Sender, "Unknown database", ctx); + } + TxProcessor->ProcessTx(CreateTxReplaceDatabaseYamlConfig(ev), ctx); + }, + [&](const NYamlConfig::TError& error) { + AuditLogReplaceConfigTransaction( + /* peer = */ ev->Get()->Record.GetPeerName(), + /* userSID = */ NACLib::TUserToken(ev->Get()->Record.GetUserToken()).GetUserSID(), + /* sanitizedToken = */ NACLib::TUserToken(ev->Get()->Record.GetUserToken()).GetSanitizedToken(), + /* oldConfig = */ YamlConfig, + /* newConfig = */ ev->Get()->Record.GetRequest().config(), + /* reason = */ error.Error, + /* success = */ false); + return FailReplaceConfig(ev->Sender, error.Error, ctx); + } + }, metadata); } void TConfigsManager::Handle(TEvConsole::TEvSetYamlConfigRequest::TPtr &ev, const TActorContext &ctx) { - TxProcessor->ProcessTx(CreateTxSetYamlConfig(ev), ctx); + auto metadata = NYamlConfig::GetGenericMetadata(ev->Get()->Record.GetRequest().config()); + + std::visit(TOverloaded{ + [&](const NYamlConfig::TMainMetadata& /* value */) { + TxProcessor->ProcessTx(CreateTxSetMainYamlConfig(ev), ctx); + }, + [&](const NYamlConfig::TDatabaseMetadata& value) { + if (!value.Database || !Self.HasTenant(*value.Database)) { + return FailReplaceConfig(ev->Sender, "Unknown database", ctx); + } + TxProcessor->ProcessTx(CreateTxSetDatabaseYamlConfig(ev), ctx); + }, + [&](const NYamlConfig::TError& error) { + AuditLogReplaceConfigTransaction( + /* peer = */ ev->Get()->Record.GetPeerName(), + /* userSID = */ NACLib::TUserToken(ev->Get()->Record.GetUserToken()).GetUserSID(), + /* sanitizedToken = */ NACLib::TUserToken(ev->Get()->Record.GetUserToken()).GetSanitizedToken(), + /* oldConfig = */ YamlConfig, + /* newConfig = */ ev->Get()->Record.GetRequest().config(), + /* reason = */ error.Error, + /* success = */ false); + return FailReplaceConfig(ev->Sender, error.Error, ctx); + } + }, metadata); +} + +void TConfigsManager::FailReplaceConfig(TActorId Sender, const TString& error, const TActorContext &ctx) { + auto resp = MakeHolder(); + resp->Record.SetYdbStatus(Ydb::StatusIds::BAD_REQUEST); + auto *issue = resp->Record.AddIssues(); + issue->set_severity(NYql::TSeverityIds::S_ERROR); + issue->set_message(error); + auto response = MakeHolder(Sender, ctx.SelfID, resp.Release()); + ctx.Send(response.Release()); } void TConfigsManager::Handle(TEvConsole::TEvDropConfigRequest::TPtr &ev, const TActorContext &ctx) @@ -989,7 +1129,7 @@ void TConfigsManager::Handle(TEvPrivate::TEvStateLoaded::TPtr &/*ev*/, const TAc ctx.Send(ConfigsProvider, new TConfigsProvider::TEvPrivate::TEvSetSubscriptions(SubscriptionIndex.GetSubscriptions())); ctx.Send(GetNameserviceActorId(), new TEvInterconnect::TEvListNodes()); if (!YamlConfig.empty()) { - ctx.Send(ConfigsProvider, new TConfigsProvider::TEvPrivate::TEvUpdateYamlConfig(YamlConfig, VolatileYamlConfigs)); + ctx.Send(ConfigsProvider, new TConfigsProvider::TEvPrivate::TEvUpdateYamlConfig(YamlConfig, YamlConfigPerDatabase, VolatileYamlConfigs)); } ScheduleLogCleanup(ctx); } diff --git a/ydb/core/cms/console/console_configs_manager.h b/ydb/core/cms/console/console_configs_manager.h index a95513965ec5..5763f096960c 100644 --- a/ydb/core/cms/console/console_configs_manager.h +++ b/ydb/core/cms/console/console_configs_manager.h @@ -32,16 +32,27 @@ class TConfigsManager : public TActorBootstrapped { using TBase = TActorBootstrapped; - struct TValidateConfigResult { - std::optional ErrorReason; - bool Modify = false; + struct TUpdateConfigOpBaseContext { + std::optional Error; + + TMap> DeprecatedFields; + TMap> UnknownFields; + }; + + struct TUpdateConfigOpContext + : public TUpdateConfigOpBaseContext + { TString UpdatedConfig; ui32 Version; TString Cluster; - bool HasForbiddenUnknown = false; - TMap> DeprecatedFields; - TMap> UnknownFields; - bool ValidationFinished = false; + }; + + struct TUpdateDatabaseConfigOpContext + : public TUpdateConfigOpBaseContext + { + TString UpdatedConfig; + TString TargetDatabase; + ui32 Version; }; public: @@ -69,7 +80,13 @@ class TConfigsManager : public TActorBootstrapped { bool CheckConfig(const NKikimrConsole::TConfigsConfig &config, Ydb::StatusIds::StatusCode &code, TString &error); - TValidateConfigResult ValidateConfigAndReplaceMetadata(const TString& config, bool force = false, bool allowUnknownFields = false); + + + void ReplaceMainConfigMetadata(const TString &config, bool force, TUpdateConfigOpContext& opCtx); + void ValidateMainConfig(TUpdateConfigOpContext& opCtx); + + void ReplaceDatabaseConfigMetadata(const TString &config, bool force, TUpdateDatabaseConfigOpContext& opCtx); + void ValidateDatabaseConfig(TUpdateDatabaseConfigOpContext& opCtx); void SendInReply(const TActorId& sender, const TActorId& icSession, std::unique_ptr ev, ui64 cookie = 0); @@ -128,7 +145,9 @@ class TConfigsManager : public TActorBootstrapped { class TTxUpdateLastProvidedConfig; class TTxGetLogTail; class TTxLogCleanup; - class TTxReplaceYamlConfig; + class TTxReplaceYamlConfigBase; + class TTxReplaceMainYamlConfig; + class TTxReplaceDatabaseYamlConfig; class TTxDropYamlConfig; class TTxGetYamlConfig; class TTxGetYamlMetadata; @@ -145,8 +164,10 @@ class TConfigsManager : public TActorBootstrapped { ITransaction *CreateTxUpdateLastProvidedConfig(TEvConsole::TEvConfigNotificationResponse::TPtr &ev); ITransaction *CreateTxGetLogTail(TEvConsole::TEvGetLogTailRequest::TPtr &ev); ITransaction *CreateTxLogCleanup(); - ITransaction *CreateTxReplaceYamlConfig(TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev); - ITransaction *CreateTxSetYamlConfig(TEvConsole::TEvSetYamlConfigRequest::TPtr &ev); + ITransaction *CreateTxReplaceMainYamlConfig(TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev); + ITransaction *CreateTxReplaceDatabaseYamlConfig(TEvConsole::TEvReplaceYamlConfigRequest::TPtr &ev); + ITransaction *CreateTxSetMainYamlConfig(TEvConsole::TEvSetYamlConfigRequest::TPtr &ev); + ITransaction *CreateTxSetDatabaseYamlConfig(TEvConsole::TEvSetYamlConfigRequest::TPtr &ev); ITransaction *CreateTxDropYamlConfig(TEvConsole::TEvDropConfigRequest::TPtr &ev); ITransaction *CreateTxGetYamlConfig(TEvConsole::TEvGetAllConfigsRequest::TPtr &ev); ITransaction *CreateTxGetYamlMetadata(TEvConsole::TEvGetAllMetadataRequest::TPtr &ev); @@ -181,6 +202,8 @@ class TConfigsManager : public TActorBootstrapped { void Handle(TEvBlobStorage::TEvControllerConsoleCommitRequest::TPtr &ev, const TActorContext &ctx); void Handle(TEvBlobStorage::TEvControllerValidateConfigRequest::TPtr &ev, const TActorContext &ctx); + void FailReplaceConfig(TActorId Sender, const TString& error, const TActorContext &ctx); + static bool CheckRights(const TString& userToken); template @@ -301,6 +324,7 @@ class TConfigsManager : public TActorBootstrapped { TString ClusterName; ui32 YamlVersion = 0; TString YamlConfig; + THashMap YamlConfigPerDatabase; bool YamlDropped = false; bool YamlReadOnly = true; TMap VolatileYamlConfigs; diff --git a/ydb/core/cms/console/console_configs_provider.cpp b/ydb/core/cms/console/console_configs_provider.cpp index 1f5691941c63..603c9177fa72 100644 --- a/ydb/core/cms/console/console_configs_provider.cpp +++ b/ydb/core/cms/console/console_configs_provider.cpp @@ -709,6 +709,15 @@ void TConfigsProvider::CheckSubscription(TInMemorySubscription::TPtr subscriptio subscription->VolatileYamlConfigHashes = VolatileYamlConfigHashes; + if (auto it = YamlConfigPerDatabase.find(subscription->Tenant); it != YamlConfigPerDatabase.end()) { + if (!subscription->DatabaseYamlConfigVersion || *subscription->DatabaseYamlConfigVersion != it->second.Version) { + subscription->DatabaseYamlConfigVersion = it->second.Version; + request->Record.SetDatabaseConfig(it->second.Config); + } else { + request->Record.SetDatabaseConfigNotChanged(true); + } + } + ctx.Send(subscription->Worker, request.Release()); subscription->FirstUpdateSent = true; @@ -1099,6 +1108,10 @@ void TConfigsProvider::Handle(TEvConsole::TEvGetNodeConfigRequest::TPtr &ev, con item.SetId(id); item.SetConfig(config); } + + if (auto it = YamlConfigPerDatabase.find(rec.GetNode().GetTenant()); it != YamlConfigPerDatabase.end()) { + response->Record.SetDatabaseYamlConfig(it->second.Config); + } } ctx.Send(ev->Sender, response.Release(), 0, ev->Cookie); @@ -1223,49 +1236,76 @@ void TConfigsProvider::Handle(TEvPrivate::TEvUpdateSubscriptions::TPtr &ev, cons } void TConfigsProvider::Handle(TEvPrivate::TEvUpdateYamlConfig::TPtr &ev, const TActorContext &ctx) { - YamlConfig = ev->Get()->YamlConfig; - VolatileYamlConfigs.clear(); - - YamlConfigVersion = NYamlConfig::GetVersion(YamlConfig); - VolatileYamlConfigHashes.clear(); - for (auto& [id, config] : ev->Get()->VolatileYamlConfigs) { - auto doc = NFyaml::TDocument::Parse(config); - // we strip it to provide old format for config dispatcher - auto node = doc.Root().Map().at("selector_config"); - TString strippedConfig = "\n" + config.substr(node.BeginMark().InputPos, node.EndMark().InputPos - node.BeginMark().InputPos) + "\n"; - VolatileYamlConfigs[id] = strippedConfig; - VolatileYamlConfigHashes[id] = THash()(strippedConfig); - } - - for (auto &[_, subscription] : InMemoryIndex.GetSubscriptions()) { - if (subscription->ServeYaml) { - auto request = MakeHolder( - subscription->Generation, - NKikimrConfig::TAppConfig{}, - THashSet{}); - - if (subscription->YamlConfigVersion != YamlConfigVersion) { - subscription->YamlConfigVersion = YamlConfigVersion; - request->Record.SetYamlConfig(YamlConfig); - } else { - request->Record.SetYamlConfigNotChanged(true); - } + YamlConfigPerDatabase = ev->Get()->YamlConfigPerDatabase; + if (!ev->Get()->ChangedDatabase) { + YamlConfig = ev->Get()->YamlConfig; + VolatileYamlConfigs.clear(); + + YamlConfigVersion = NYamlConfig::GetVersion(YamlConfig); + VolatileYamlConfigHashes.clear(); + + for (auto& [id, config] : ev->Get()->VolatileYamlConfigs) { + auto doc = NFyaml::TDocument::Parse(config); + // we strip it to provide old format for config dispatcher + auto node = doc.Root().Map().at("selector_config"); + TString strippedConfig = "\n" + config.substr(node.BeginMark().InputPos, node.EndMark().InputPos - node.BeginMark().InputPos) + "\n"; + VolatileYamlConfigs[id] = strippedConfig; + VolatileYamlConfigHashes[id] = THash()(strippedConfig); + } - for (auto &[id, hash] : VolatileYamlConfigHashes) { - auto *volatileConfig = request->Record.AddVolatileConfigs(); - volatileConfig->SetId(id); - if (auto it = subscription->VolatileYamlConfigHashes.find(id); it != subscription->VolatileYamlConfigHashes.end() && it->second == hash) { - volatileConfig->SetNotChanged(true); - } else { - volatileConfig->SetConfig(VolatileYamlConfigs[id]); - } + for (auto &[_, subscription] : InMemoryIndex.GetSubscriptions()) { + UpdateConfig(subscription, ctx); + } + } else { + const auto* subs = InMemoryIndex.GetSubscriptions(ev->Get()->ChangedDatabase); + if (subs) { + for (auto &subscription : *subs) { + UpdateConfig(subscription, ctx); } + } + } +} - subscription->VolatileYamlConfigHashes = VolatileYamlConfigHashes; +void TConfigsProvider::UpdateConfig(TInMemorySubscription::TPtr subscription, + const TActorContext &ctx) +{ + if (subscription->ServeYaml) { + auto request = MakeHolder( + subscription->Generation, + NKikimrConfig::TAppConfig{}, + THashSet{}); - ctx.Send(subscription->Worker, request.Release()); + if (subscription->YamlConfigVersion != YamlConfigVersion) { + subscription->YamlConfigVersion = YamlConfigVersion; + request->Record.SetYamlConfig(YamlConfig); + } else { + request->Record.SetYamlConfigNotChanged(true); + } + + for (auto &[id, hash] : VolatileYamlConfigHashes) { + auto *volatileConfig = request->Record.AddVolatileConfigs(); + volatileConfig->SetId(id); + if (auto it = subscription->VolatileYamlConfigHashes.find(id); it != subscription->VolatileYamlConfigHashes.end() && it->second == hash) { + volatileConfig->SetNotChanged(true); + } else { + volatileConfig->SetConfig(VolatileYamlConfigs[id]); } } + + subscription->VolatileYamlConfigHashes = VolatileYamlConfigHashes; + + if (auto it = YamlConfigPerDatabase.find(subscription->Tenant); it != YamlConfigPerDatabase.end()) { + if (!subscription->DatabaseYamlConfigVersion || *subscription->DatabaseYamlConfigVersion != it->second.Version) { + subscription->DatabaseYamlConfigVersion = it->second.Version; + request->Record.SetDatabaseConfig(it->second.Config); + } else { + request->Record.SetDatabaseConfigNotChanged(true); + } + } + + ctx.Send(subscription->Worker, request.Release()); +} + } } // namespace NKikimr::NConsole diff --git a/ydb/core/cms/console/console_configs_provider.h b/ydb/core/cms/console/console_configs_provider.h index 7f6c18e466bb..a6a45f02c87e 100644 --- a/ydb/core/cms/console/console_configs_provider.h +++ b/ydb/core/cms/console/console_configs_provider.h @@ -13,6 +13,11 @@ namespace NKikimr::NConsole { +struct TDatabaseYamlConfig { + TString Config; + ui32 Version; +}; + class TConfigsProvider : public TActorBootstrapped { public: struct TEvPrivate { @@ -100,14 +105,48 @@ class TConfigsProvider : public TActorBootstrapped { }; struct TEvUpdateYamlConfig : public TEventLocal { - TEvUpdateYamlConfig(const TString &yamlConfig, const TMap &volatileYamlConfigs = {}) + TEvUpdateYamlConfig( + const TString &yamlConfig, + const TMap &volatileYamlConfigs = {}) : YamlConfig(yamlConfig) , VolatileYamlConfigs(volatileYamlConfigs) { } + TEvUpdateYamlConfig( + const TString &yamlConfig, + const THashMap &yamlConfigPerDatabase) + : YamlConfig(yamlConfig) + , YamlConfigPerDatabase(yamlConfigPerDatabase) + { + } + + TEvUpdateYamlConfig( + const TString &yamlConfig, + const THashMap &yamlConfigPerDatabase, + const TMap &volatileYamlConfigs) + : YamlConfig(yamlConfig) + , YamlConfigPerDatabase(yamlConfigPerDatabase) + , VolatileYamlConfigs(volatileYamlConfigs) + { + } + + TEvUpdateYamlConfig( + const TString &yamlConfig, + const THashMap &yamlConfigPerDatabase, + const TMap &volatileYamlConfigs, + const TString& changedDatabase) + : YamlConfig(yamlConfig) + , YamlConfigPerDatabase(yamlConfigPerDatabase) + , VolatileYamlConfigs(volatileYamlConfigs) + , ChangedDatabase(changedDatabase) + { + } + TString YamlConfig; + THashMap YamlConfigPerDatabase; TMap VolatileYamlConfigs; + TString ChangedDatabase; }; struct TEvUpdateSubscriptions : public TEventLocal { @@ -142,6 +181,9 @@ class TConfigsProvider : public TActorBootstrapped { void CheckSubscription(TInMemorySubscription::TPtr subscriptions, const TActorContext &ctx); + void UpdateConfig(TInMemorySubscription::TPtr subscription, + const TActorContext &ctx); + void Handle(NMon::TEvHttpInfo::TPtr &ev); void Handle(TEvConsole::TEvConfigSubscriptionRequest::TPtr &ev, const TActorContext &ctx); void Handle(TEvConsole::TEvConfigSubscriptionCanceled::TPtr &ev, const TActorContext &ctx); @@ -229,6 +271,7 @@ class TConfigsProvider : public TActorBootstrapped { TString YamlConfig; TMap VolatileYamlConfigs; + THashMap YamlConfigPerDatabase; ui64 YamlConfigVersion = 0; TMap VolatileYamlConfigHashes; }; diff --git a/ydb/core/cms/console/console_configs_subscriber.cpp b/ydb/core/cms/console/console_configs_subscriber.cpp index f69ab7307a87..3496e62fcc50 100644 --- a/ydb/core/cms/console/console_configs_subscriber.cpp +++ b/ydb/core/cms/console/console_configs_subscriber.cpp @@ -220,6 +220,9 @@ class TConfigsSubscriber : public TActorBootstrapped { bool notChanged = true; if (ServeYaml) { + if (!(rec.HasDatabaseConfigNotChanged() && rec.GetDatabaseConfigNotChanged())) { + notChanged = false; + } if (!(rec.HasYamlConfigNotChanged() && rec.GetYamlConfigNotChanged())) { if (rec.HasYamlConfig()) { YamlConfig = rec.GetYamlConfig(); @@ -285,7 +288,8 @@ class TConfigsSubscriber : public TActorBootstrapped { changes, YamlConfig, VolatileYamlConfigs, - CurrentDynConfig), + CurrentDynConfig, + rec.HasDatabaseConfig() ? TMaybe(rec.GetDatabaseConfig()) : TMaybe{}), IEventHandle::FlagTrackDelivery, Cookie); FirstUpdateSent = true; diff --git a/ydb/core/cms/console/console_handshake.cpp b/ydb/core/cms/console/console_handshake.cpp index 43605b5ad391..31dc8dc7ba6c 100644 --- a/ydb/core/cms/console/console_handshake.cpp +++ b/ydb/core/cms/console/console_handshake.cpp @@ -132,20 +132,22 @@ void TConfigsManager::Handle(TEvBlobStorage::TEvControllerValidateConfigRequest: if (!CheckSession(*ev, response, NKikimrBlobStorage::TEvControllerValidateConfigResponse::IdPipeServerMismatch)) { return; } - auto result = ValidateConfigAndReplaceMetadata(yamlConfig); - if (result.ErrorReason || result.HasForbiddenUnknown) { + TUpdateConfigOpContext opCtx; + ReplaceMainConfigMetadata(yamlConfig, false, opCtx); + ValidateMainConfig(opCtx); + if (opCtx.Error || !opCtx.UnknownFields.empty()) { record.SetStatus(NKikimrBlobStorage::TEvControllerValidateConfigResponse::ConfigNotValid); - if (!result.ErrorReason) { + if (!opCtx.Error) { record.SetErrorReason("has forbidden unknown fields"); } else { - record.SetErrorReason(*result.ErrorReason); - if (result.HasForbiddenUnknown) { + record.SetErrorReason(*opCtx.Error); + if (!opCtx.UnknownFields.empty()) { record.SetErrorReason(record.GetErrorReason() + " + has forbidden unknown fields"); } } } else { record.SetStatus(NKikimrBlobStorage::TEvControllerValidateConfigResponse::ConfigIsValid); - record.SetYAML(result.UpdatedConfig); + record.SetYAML(opCtx.UpdatedConfig); } SendInReply(ev->Sender, ev->InterconnectSession, std::move(response), ev->Cookie); } diff --git a/ydb/core/cms/console/console_impl.h b/ydb/core/cms/console/console_impl.h index eb4432ca0eb6..a5de7325048f 100644 --- a/ydb/core/cms/console/console_impl.h +++ b/ydb/core/cms/console/console_impl.h @@ -179,6 +179,8 @@ class TConsole : public TActor return Config; } + bool HasTenant(const TString& path) const; + private: TDeque> InitQueue; NKikimrConsole::TConfig Config; diff --git a/ydb/core/cms/console/console_tenants_manager.cpp b/ydb/core/cms/console/console_tenants_manager.cpp index bca83aa8e8a7..e0e2f6e00639 100644 --- a/ydb/core/cms/console/console_tenants_manager.cpp +++ b/ydb/core/cms/console/console_tenants_manager.cpp @@ -1627,7 +1627,7 @@ TTenantsManager::TTenant::TPtr TTenantsManager::FindComputationalUnitKindUsage(c return nullptr; } -TTenantsManager::TTenant::TPtr TTenantsManager::GetTenant(const TString &name) +TTenantsManager::TTenant::TPtr TTenantsManager::GetTenant(const TString &name) const { auto it = Tenants.find(name); if (it != Tenants.end()) @@ -1635,7 +1635,7 @@ TTenantsManager::TTenant::TPtr TTenantsManager::GetTenant(const TString &name) return nullptr; } -TTenantsManager::TTenant::TPtr TTenantsManager::GetTenant(const TDomainId &domainId) +TTenantsManager::TTenant::TPtr TTenantsManager::GetTenant(const TDomainId &domainId) const { auto it = TenantIdToName.find(domainId); if (it != TenantIdToName.end()) diff --git a/ydb/core/cms/console/console_tenants_manager.h b/ydb/core/cms/console/console_tenants_manager.h index e1af8f391f15..04c92558821b 100644 --- a/ydb/core/cms/console/console_tenants_manager.h +++ b/ydb/core/cms/console/console_tenants_manager.h @@ -756,8 +756,8 @@ class TTenantsManager : public TActorBootstrapped { TTenant::TPtr FindComputationalUnitKindUsage(const TString &kind); TTenant::TPtr FindComputationalUnitKindUsage(const TString &kind, const TString &zone); - TTenant::TPtr GetTenant(const TString &name); - TTenant::TPtr GetTenant(const TDomainId &domainId); + TTenant::TPtr GetTenant(const TString &name) const; + TTenant::TPtr GetTenant(const TDomainId &domainId) const; void AddTenant(TTenant::TPtr tenant); void RemoveTenant(TTenant::TPtr tenant); void RemoveTenantFailed(TTenant::TPtr tenant, @@ -1006,6 +1006,9 @@ class TTenantsManager : public TActorBootstrapped { void Bootstrap(const TActorContext &ctx); void Detach(); + bool HasTenant(const TString& path) const { + return Tenants.contains(path); + } private: TConsole &Self; diff --git a/ydb/core/cms/console/util/config_index.cpp b/ydb/core/cms/console/util/config_index.cpp index 75d620fbaa1a..57dd50e63ddb 100644 --- a/ydb/core/cms/console/util/config_index.cpp +++ b/ydb/core/cms/console/util/config_index.cpp @@ -701,6 +701,15 @@ const THashMap &TInMemorySubscriptionInde return Subscriptions; } +const TInMemorySubscriptionSet *TInMemorySubscriptionIndex::GetSubscriptions(const TString& path) const +{ + if (auto it = SubscriptionsByTenant.find(path); it != SubscriptionsByTenant.end()) { + return &(it->second); + } + + return nullptr; +} + void TInMemorySubscriptionIndex::AddSubscription(TInMemorySubscription::TPtr subscription) { Y_ABORT_UNLESS(!Subscriptions.contains(subscription->Subscriber)); diff --git a/ydb/core/cms/console/util/config_index.h b/ydb/core/cms/console/util/config_index.h index 821f9482bd8e..71fa72ca1a6e 100644 --- a/ydb/core/cms/console/util/config_index.h +++ b/ydb/core/cms/console/util/config_index.h @@ -669,6 +669,7 @@ struct TInMemorySubscription : public TThrRefBase { ui64 YamlApiVersion = 0; ui64 YamlConfigVersion = 0; TMap VolatileYamlConfigHashes; + std::optional DatabaseYamlConfigVersion; bool FirstUpdateSent = false; @@ -684,6 +685,7 @@ class TInMemorySubscriptionIndex { TInMemorySubscription::TPtr GetSubscription(const TActorId &subscriber); const THashMap &GetSubscriptions() const; + const TInMemorySubscriptionSet *GetSubscriptions(const TString &path) const; void AddSubscription(TInMemorySubscription::TPtr subscription); diff --git a/ydb/core/config/init/init.cpp b/ydb/core/config/init/init.cpp index 574ee1081f7c..831ffed06d98 100644 --- a/ydb/core/config/init/init.cpp +++ b/ydb/core/config/init/init.cpp @@ -318,21 +318,29 @@ class TDynConfigResultWrapper : Result(std::move(result)) {} - const NKikimrConfig::TAppConfig& GetConfig() const { + const NKikimrConfig::TAppConfig& GetConfig() const override { return Result.GetConfig(); } - bool HasYamlConfig() const { + bool HasYamlConfig() const override { return Result.HasYamlConfig(); } - const TString& GetYamlConfig() const { + const TString& GetYamlConfig() const override { return Result.GetYamlConfig(); } - TMap GetVolatileYamlConfigs() const { + TMap GetVolatileYamlConfigs() const override { return Result.GetVolatileYamlConfigs(); } + + bool HasDatabaseYamlConfig() const override { + return Result.HasDatabaseYamlConfig(); + } + + const TString& GetDatabaseYamlConfig() const override { + return Result.GetDatabaseYamlConfig(); + } }; class TDefaultDynConfigClient @@ -691,7 +699,8 @@ NKikimrConfig::TAppConfig GetYamlConfigFromResult(const IConfigurationResult& re result.GetYamlConfig(), result.GetVolatileYamlConfigs(), labels, - yamlConfig); + yamlConfig, + result.HasDatabaseYamlConfig() ? std::optional{result.GetDatabaseYamlConfig()} : std::nullopt); } return yamlConfig; } diff --git a/ydb/core/config/init/init.h b/ydb/core/config/init/init.h index 9786ee5eba85..fb10a0864120 100644 --- a/ydb/core/config/init/init.h +++ b/ydb/core/config/init/init.h @@ -159,6 +159,8 @@ class IConfigurationResult { virtual bool HasYamlConfig() const = 0; virtual const TString& GetYamlConfig() const = 0; virtual TMap GetVolatileYamlConfigs() const = 0; + virtual bool HasDatabaseYamlConfig() const = 0; + virtual const TString& GetDatabaseYamlConfig() const = 0; }; class IDynConfigClient { diff --git a/ydb/core/driver_lib/cli_base/cli_cmds_root.cpp b/ydb/core/driver_lib/cli_base/cli_cmds_root.cpp index dc7a92f11ddc..a42c7cb20a7d 100644 --- a/ydb/core/driver_lib/cli_base/cli_cmds_root.cpp +++ b/ydb/core/driver_lib/cli_base/cli_cmds_root.cpp @@ -66,6 +66,8 @@ void TClientCommandRootKikimrBase::Parse(TConfig& config) { ParseProfile(); GetProfileVariable("path", config.Path); TClientCommandRootBase::Parse(config); + ParseCredentials(config); + ParseAddress(config); NClient::TKikimr::DUMP_REQUESTS = DumpRequests; } diff --git a/ydb/core/grpc_services/rpc_dynamic_config.cpp b/ydb/core/grpc_services/rpc_dynamic_config.cpp index b779e6283d73..2ed2c64578fd 100644 --- a/ydb/core/grpc_services/rpc_dynamic_config.cpp +++ b/ydb/core/grpc_services/rpc_dynamic_config.cpp @@ -206,6 +206,9 @@ class TDynamicConfigRPC : public TRpcOperationRequestActorRecord.MutableRequest()->CopyFrom(*this->GetProtoRequest()); request->Record.SetUserToken(this->Request_->GetSerializedToken()); request->Record.SetPeerName(this->Request_->GetPeerName()); + if (this->Request_->GetDatabaseName()) { + request->Record.SetDatabase(*this->Request_->GetDatabaseName()); + } NTabletPipe::SendData(IActor::SelfId(), ConsolePipe, request.Release()); } }; diff --git a/ydb/core/protos/console.proto b/ydb/core/protos/console.proto index e6d83427afc5..b7ca6afdd1f8 100644 --- a/ydb/core/protos/console.proto +++ b/ydb/core/protos/console.proto @@ -38,6 +38,12 @@ message TYamlConfigChange { repeated TVolatileConfig NewVolatileYamlConfigs = 4; } +message TDatabaseConfigChange { + optional string Database = 1; + optional string OldYamlConfig = 2; + optional string NewYamlConfig = 3; +} + message TLogRecordData { // Old configs optional TConfigureRequest Action = 1; @@ -45,6 +51,7 @@ message TLogRecordData { repeated uint32 AffectedKinds = 3; // New configs optional TYamlConfigChange YamlConfigChange = 4; + optional TDatabaseConfigChange DatabaseConfigChange = 5; } message TLogRecord { diff --git a/ydb/core/protos/console_config.proto b/ydb/core/protos/console_config.proto index b161c98de09b..9bba68aac481 100644 --- a/ydb/core/protos/console_config.proto +++ b/ydb/core/protos/console_config.proto @@ -147,7 +147,8 @@ message TConfigItem { NamedConfigsItem = 100; ClusterYamlConfigItem = 101; - // synthetic kind for audit purposes only + // synthetic kinds for audit purposes only + DatabaseYamlConfigChangeItem = 32767; YamlConfigChangeItem = 32768; } @@ -263,6 +264,7 @@ message TSetYamlConfigRequest { optional Ydb.DynamicConfig.SetConfigRequest Request = 1; optional bytes UserToken = 2; optional string PeerName = 3; + optional string Database = 4; } message TSetYamlConfigResponse { @@ -273,6 +275,7 @@ message TReplaceYamlConfigRequest { optional Ydb.DynamicConfig.ReplaceConfigRequest Request = 1; optional bytes UserToken = 2; optional string PeerName = 3; + optional string Database = 4; } message TReplaceYamlConfigResponse { @@ -283,6 +286,7 @@ message TDropConfigRequest { optional Ydb.DynamicConfig.DropConfigRequest Request = 1; optional bytes UserToken = 2; optional string PeerName = 3; + optional string Database = 4; } message TDropConfigResponse { @@ -292,6 +296,7 @@ message TAddVolatileConfigRequest { optional Ydb.DynamicConfig.AddVolatileConfigRequest Request = 1; optional bytes UserToken = 2; optional string PeerName = 3; + optional string Database = 4; } message TAddVolatileConfigResponse { @@ -301,6 +306,7 @@ message TGetAllConfigsRequest { optional Ydb.DynamicConfig.GetConfigRequest Request = 1; optional bytes UserToken = 2; optional string PeerName = 3; + optional string Database = 4; } message TGetAllConfigsResponse { @@ -311,6 +317,7 @@ message TGetAllConfigsResponse { message TIsYamlReadOnlyRequest { optional bytes UserToken = 1; optional string PeerName = 3; + optional string Database = 4; } message TIsYamlReadOnlyResponse { @@ -321,6 +328,7 @@ message TGetNodeLabelsRequest { optional Ydb.DynamicConfig.GetNodeLabelsRequest Request = 1; optional bytes UserToken = 2; optional string PeerName = 3; + optional string Database = 4; } message TGetNodeLabelsResponse { @@ -331,6 +339,7 @@ message TGetAllMetadataRequest { optional Ydb.DynamicConfig.GetMetadataRequest Request = 1; optional bytes UserToken = 2; optional string PeerName = 3; + optional string Database = 4; } message TGetAllMetadataResponse { @@ -341,6 +350,7 @@ message TRemoveVolatileConfigRequest { optional Ydb.DynamicConfig.RemoveVolatileConfigRequest Request = 1; optional bytes UserToken = 2; optional string PeerName = 3; + optional string Database = 4; } message TRemoveVolatileConfigResponse { @@ -350,6 +360,7 @@ message TResolveConfigRequest { optional Ydb.DynamicConfig.ResolveConfigRequest Request = 1; optional bytes UserToken = 2; optional string PeerName = 3; + optional string Database = 4; } message TResolveConfigResponse { @@ -360,6 +371,7 @@ message TResolveAllConfigRequest { optional Ydb.DynamicConfig.ResolveAllConfigRequest Request = 1; optional bytes UserToken = 2; optional string PeerName = 3; + optional string Database = 4; } message TResolveAllConfigResponse { @@ -449,6 +461,7 @@ message TGetNodeConfigResponse { optional NKikimrConfig.TAppConfig Config = 2; optional string YamlConfig = 3; repeated TVolatileYamlConfig VolatileConfigs = 4; + optional string DatabaseYamlConfig = 5; } message TSubscriber { @@ -503,6 +516,8 @@ message TConfigSubscriptionNotification { optional string YamlConfig = 5; optional bool YamlConfigNotChanged = 6; repeated TVolatileYamlConfig VolatileConfigs = 7; + optional string DatabaseConfig = 8; + optional bool DatabaseConfigNotChanged = 9; optional NKikimrConfig.TAppConfig RawConsoleConfig = 103; // Used for correct next startup config prediction } diff --git a/ydb/core/protos/feature_flags.proto b/ydb/core/protos/feature_flags.proto index 8553c96c292e..076d61905031 100644 --- a/ydb/core/protos/feature_flags.proto +++ b/ydb/core/protos/feature_flags.proto @@ -188,4 +188,5 @@ message TFeatureFlags { optional bool EnableTopicTransfer = 163 [default = false]; optional bool EnableViewExport = 164 [default = false]; optional bool EnableColumnStore = 165 [default = false]; + optional bool PerDatabaseConfigAllowed = 166 [default = false]; } diff --git a/ydb/core/viewer/viewer_feature_flags.h b/ydb/core/viewer/viewer_feature_flags.h index 71000dd00cf5..37233da083d8 100644 --- a/ydb/core/viewer/viewer_feature_flags.h +++ b/ydb/core/viewer/viewer_feature_flags.h @@ -157,9 +157,9 @@ class TJsonFeatureFlags : public TViewerPipeClient { } } NKikimrConfig::TAppConfig appConfig; - if (AllConfigsResponse->Record.GetResponse().config()) { + if (AllConfigsResponse->Record.GetResponse().config_size()) { try { - NYamlConfig::ResolveAndParseYamlConfig(AllConfigsResponse->Record.GetResponse().config(), {}, {{"tenant", realDatabase}}, appConfig); + NYamlConfig::ResolveAndParseYamlConfig(AllConfigsResponse->Record.GetResponse().config(0), {}, {{"tenant", realDatabase}}, appConfig); } catch (const std::exception& e) { BLOG_ERROR("Failed to parse config for tenant " << realDatabase << ": " << e.what()); } diff --git a/ydb/library/yaml_config/public/yaml_config.cpp b/ydb/library/yaml_config/public/yaml_config.cpp index 89ba29283908..cb0ff559cda5 100644 --- a/ydb/library/yaml_config/public/yaml_config.cpp +++ b/ydb/library/yaml_config/public/yaml_config.cpp @@ -622,12 +622,29 @@ void AppendVolatileConfigs(NFyaml::TDocument& config, NFyaml::TNodeRef& volatile } } +void AppendDatabaseConfig(NFyaml::TDocument& config, NFyaml::TDocument& databaseConfig) { + auto configRoot = config.Root(); + if (!configRoot.Map().Has("selector_config")) { + configRoot.Map().Append(config.Buildf("selector_config"), config.Buildf("[]")); + } + + auto databaseConfigRoot = databaseConfig.Root(); + + auto selectors = configRoot.Map().at("selector_config").Sequence(); + selectors.Append(config.Buildf(R"( +description: Implicit DatabaseConfig node +selector: {} +)")); + auto node = databaseConfigRoot.Copy(config); + selectors.at(0).Map().Append(config.Buildf("config"), node); +} + ui64 GetVersion(const TString& config) { auto metadata = GetMetadata(config); return metadata.Version.value_or(0); } -TMetadata GetMetadata(const TString& config) { +TMainMetadata GetMetadata(const TString& config) { if (config.empty()) { return {}; } @@ -637,7 +654,7 @@ TMetadata GetMetadata(const TString& config) { if (auto node = doc.Root().Map()["metadata"]; node) { auto versionNode = node.Map()["version"]; auto clusterNode = node.Map()["cluster"]; - return TMetadata{ + return TMainMetadata{ .Version = versionNode ? std::optional{FromString(versionNode.Scalar())} : std::nullopt, .Cluster = clusterNode ? std::optional{clusterNode.Scalar()} : std::nullopt, }; @@ -646,6 +663,25 @@ TMetadata GetMetadata(const TString& config) { return {}; } +TDatabaseMetadata GetDatabaseMetadata(const TString& config) { + if (config.empty()) { + return {}; + } + + auto doc = NFyaml::TDocument::Parse(config); + + if (auto node = doc.Root().Map()["metadata"]; node) { + auto databaseNode = node.Map()["database"]; + auto versionNode = node.Map()["version"]; + return TDatabaseMetadata{ + .Version = versionNode ? std::optional{FromString(versionNode.Scalar())} : std::nullopt, + .Database = databaseNode ? std::optional{databaseNode.Scalar()} : std::nullopt, + }; + } + + return {}; +} + TVolatileMetadata GetVolatileMetadata(const TString& config) { if (config.empty()) { return {}; @@ -702,7 +738,7 @@ TString ReplaceMetadata(const TString& config, const std::function GetGenericMetadata(const TString& config) { + try { + auto doc = NFyaml::TDocument::Parse(config); + auto kind = doc.Root().Map().at("metadata").Map().at("kind").Scalar(); + if (kind == "MainConfig") { + return GetMetadata(config); + } else if (kind == "DatabaseConfig") { + return GetDatabaseMetadata(config); + } else { + return TError { + .Error = TString("Unknown kind: ") + kind, + }; + } + } catch (yexception& e) { + return TError { + .Error = e.what(), + }; + } +} + } // namespace NKikimr::NYamlConfig template <> diff --git a/ydb/library/yaml_config/public/yaml_config.h b/ydb/library/yaml_config/public/yaml_config.h index 157a0acd337f..48426a03faf4 100644 --- a/ydb/library/yaml_config/public/yaml_config.h +++ b/ydb/library/yaml_config/public/yaml_config.h @@ -166,6 +166,12 @@ void AppendVolatileConfigs(NFyaml::TDocument& config, NFyaml::TDocument& volatil */ void AppendVolatileConfigs(NFyaml::TDocument& config, NFyaml::TNodeRef& volatileConfig); +/** + * Appends database config to the end of selectors list + * **Important**: Document should be correct DatabaseConfig + */ +void AppendDatabaseConfig(NFyaml::TDocument& config, NFyaml::TDocument& databaseConfig); + /** * Parses config version */ @@ -174,16 +180,11 @@ ui64 GetVersion(const TString& config); /** * Represents config metadata */ -struct TMetadata { +struct TMainMetadata { std::optional Version; std::optional Cluster; }; -/** - * Parses config metadata - */ -TMetadata GetMetadata(const TString& config); - /** * Represents volatile config metadata */ @@ -193,6 +194,34 @@ struct TVolatileMetadata { std::optional Id; }; +/** + * Represents database config metadata + */ +struct TDatabaseMetadata { + // maybe we should enforce Cluster as well + std::optional Version; + std::optional Database; +}; + +struct TError { + TString Error; +}; + +/** + * Parses config metadata + */ +std::variant GetGenericMetadata(const TString& config); + +/** + * Parses config metadata + */ +TMainMetadata GetMetadata(const TString& config); + +/** + * Parses database config metadata + */ +TDatabaseMetadata GetDatabaseMetadata(const TString& config); + /** * Parses volatile config metadata */ @@ -201,7 +230,12 @@ TVolatileMetadata GetVolatileMetadata(const TString& config); /** * Replaces metadata in config */ -TString ReplaceMetadata(const TString& config, const TMetadata& metadata); +TString ReplaceMetadata(const TString& config, const TMainMetadata& metadata); + +/** + * Replaces metadata in database config + */ +TString ReplaceMetadata(const TString& config, const TDatabaseMetadata& metadata); /** * Replaces volatile metadata in config @@ -218,6 +252,11 @@ bool IsVolatileConfig(const TString& config); */ bool IsMainConfig(const TString& config); +/** + * Checks whether string is main config or not + */ +bool IsDatabaseConfig(const TString& config); + /** * Strips metadata from config */ diff --git a/ydb/library/yaml_config/yaml_config.cpp b/ydb/library/yaml_config/yaml_config.cpp index 21c4c79a95e1..651e2091b700 100644 --- a/ydb/library/yaml_config/yaml_config.cpp +++ b/ydb/library/yaml_config/yaml_config.cpp @@ -37,11 +37,17 @@ void ResolveAndParseYamlConfig( const TMap& volatileYamlConfigs, const TMap& labels, NKikimrConfig::TAppConfig& appConfig, + std::optional databaseConfig, TString* resolvedYamlConfig, TString* resolvedJsonConfig) { auto tree = NFyaml::TDocument::Parse(yamlConfig); + if (databaseConfig) { + auto d = NFyaml::TDocument::Parse(*databaseConfig); + NYamlConfig::AppendDatabaseConfig(tree, d); + } + for (auto& [_, config] : volatileYamlConfigs) { auto d = NFyaml::TDocument::Parse(config); NYamlConfig::AppendVolatileConfigs(tree, d); diff --git a/ydb/library/yaml_config/yaml_config.h b/ydb/library/yaml_config/yaml_config.h index 76064529e581..e745f733d285 100644 --- a/ydb/library/yaml_config/yaml_config.h +++ b/ydb/library/yaml_config/yaml_config.h @@ -77,6 +77,7 @@ void ResolveAndParseYamlConfig( const TMap& volatileYamlConfigs, const TMap& labels, NKikimrConfig::TAppConfig& appConfig, + std::optional databaseConfig = std::nullopt, TString* resolvedYamlConfig = nullptr, TString* resolvedJsonConfig = nullptr); diff --git a/ydb/public/api/protos/draft/ydb_dynamic_config.proto b/ydb/public/api/protos/draft/ydb_dynamic_config.proto index 9ba65d706d2f..bec05e95f7c8 100644 --- a/ydb/public/api/protos/draft/ydb_dynamic_config.proto +++ b/ydb/public/api/protos/draft/ydb_dynamic_config.proto @@ -9,8 +9,12 @@ import "ydb/public/api/protos/ydb_operation.proto"; message ConfigIdentity { // Current main config version uint64 version = 1; - // Cluster name (should be set on node with console tablet, unknown by default) - string cluster = 2; + oneof type { + // Cluster name (should be set on node with console tablet, unknown by default) + string cluster = 2; + // database name + string database = 3; + } } message SetConfigRequest { @@ -74,12 +78,12 @@ message GetConfigResponse { } message GetConfigResult { - // Main dynamic config with metadata in YAML format - string config = 1; + // Main/database dynamic config with metadata in YAML format + repeated string config = 1; // All volatile configs repeated VolatileConfig volatile_configs = 2; - ConfigIdentity identity = 3; + repeated ConfigIdentity identity = 3; } message VolatileConfigMetadata { diff --git a/ydb/public/lib/deprecated/kicli/configurator.cpp b/ydb/public/lib/deprecated/kicli/configurator.cpp index 0cf7c4f67eb8..8555d980c0cb 100644 --- a/ydb/public/lib/deprecated/kicli/configurator.cpp +++ b/ydb/public/lib/deprecated/kicli/configurator.cpp @@ -56,6 +56,16 @@ TMap TConfigurationResult::GetVolatileYamlConfigs() const return volatileConfigs; } +bool TConfigurationResult::HasDatabaseYamlConfig() const +{ + return Record().GetGetNodeConfigResponse().HasDatabaseYamlConfig(); +} + +const TString& TConfigurationResult::GetDatabaseYamlConfig() const +{ + return Record().GetGetNodeConfigResponse().GetDatabaseYamlConfig(); +} + TNodeConfigurator::TNodeConfigurator(TKikimr& kikimr) : Kikimr(&kikimr) { diff --git a/ydb/public/lib/deprecated/kicli/kicli.h b/ydb/public/lib/deprecated/kicli/kicli.h index aa46e6694bdf..cb3dd8787a10 100644 --- a/ydb/public/lib/deprecated/kicli/kicli.h +++ b/ydb/public/lib/deprecated/kicli/kicli.h @@ -693,6 +693,8 @@ class TConfigurationResult : public TResult { bool HasYamlConfig() const; const TString& GetYamlConfig() const; TMap GetVolatileYamlConfigs() const; + bool HasDatabaseYamlConfig() const; + const TString& GetDatabaseYamlConfig() const; const NKikimrClient::TConsoleResponse &Record() const; diff --git a/ydb/public/lib/ydb_cli/commands/ydb_admin.cpp b/ydb/public/lib/ydb_cli/commands/ydb_admin.cpp index 14f263455ea3..88a771bfeb60 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_admin.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_admin.cpp @@ -7,14 +7,142 @@ namespace NYdb { namespace NConsoleClient { +// FIXME: just reuse command one's +namespace { + TString FormatOption(const NLastGetopt::TOpt* option, const NColorizer::TColors& colors) { + using namespace NLastGetopt; + TStringStream result; + const TOpt::TShortNames& shorts = option->GetShortNames(); + const TOpt::TLongNames& longs = option->GetLongNames(); + + const size_t nopts = shorts.size() + longs.size(); + const bool multiple = 1 < nopts; + if (multiple) { + result << '{'; + } + for (size_t i = 0; i < nopts; ++i) { + if (multiple && 0 != i) { + result << '|'; + } + + if (i < shorts.size()) { // short + result << colors.GreenColor() << '-' << shorts[i] << colors.OldColor(); + } else { + result << colors.GreenColor() << "--" << longs[i - shorts.size()] << colors.OldColor(); + } + } + if (multiple) { + result << '}'; + } + + return result.Str(); + } + + // Option not to show in parent command help + bool NeedToHideOption(const NLastGetopt::TOpt* opt) { + if (opt->IsHidden()) { + return true; + } + for (const char shortName : opt->GetShortNames()) { + if (shortName == 'V' || shortName == 'h') + return true; + } + return false; + } + + void PrintOptionsDescription(IOutputStream& os, const NLastGetopt::TOpts* opts, NColorizer::TColors& colors, const TString& command) { + using namespace NLastGetopt; + NColorizer::TColors disabledColors(false); + os << " "; + bool firstPrintedOption = true; + for (size_t i = 0; i < opts->Opts_.size(); i++) { + const TOpt* opt = opts->Opts_[i].Get(); + if (NeedToHideOption(opt)) { + continue; + } + if (!firstPrintedOption) { + os << ", "; + } + os << FormatOption(opt, colors); + firstPrintedOption = false; + } + + os << Endl << " To get full description of these options run '" << command << "--help'."; + } + + void PrintParentOptions(TStringStream& stream, TClientCommand::TConfig& config, NColorizer::TColors& colors) { + bool foundRootParent = false; + TStringBuilder fullCommand; + for (const auto& parentCommand: config.ParentCommands) { + fullCommand << parentCommand.Name << " "; + if (parentCommand.Options) { + TString name = "Global"; + if (!foundRootParent) { + foundRootParent = true; + } else { + name = parentCommand.Name; + name[0] = toupper(name[0]); + stream << Endl << Endl; + } + stream << colors.BoldColor() << name << " options" << colors.OldColor() << ":" << Endl; + PrintOptionsDescription(stream, parentCommand.Options, colors, fullCommand); + } + } + } +} + +class TCommandNode : public TClientCommandTree { +public: + TCommandNode() + : TClientCommandTree("node", {}, "Node-wide administration") + {} +}; + +class TCommandDatabase : public TClientCommandTree { +public: + TCommandDatabase() + : TClientCommandTree("database", {}, "Database-wide administration") + { + AddCommand(std::make_unique()); + } +}; + TCommandAdmin::TCommandAdmin() : TClientCommandTree("admin", {}, "Administrative cluster operations") { - AddCommand(std::make_unique()); - AddCommand(std::make_unique()); - AddCommand(std::make_unique()); + MarkDangerous(); + UseOnlyExplicitProfile(); + AddHiddenCommand(std::make_unique(false)); + AddHiddenCommand(std::make_unique()); + AddHiddenCommand(std::make_unique(false)); AddCommand(std::make_unique()); + AddCommand(std::make_unique()); + AddCommand(std::make_unique()); } +void TCommandAdmin::Config(TConfig& config) { + TClientCommand::Config(config); + SetFreeArgs(config); + TString commands; + SetFreeArgTitle(0, "", commands); + TStringStream stream; + NColorizer::TColors colors = NColorizer::AutoColors(Cout); + stream << Endl << Endl + << colors.BoldColor() + << "Commands in this subtree may damage your cluster if used wrong" << Endl + << "Due to dangerous nature of this commands ALL global parameters must be set explicitly" << Endl + << "Profiles are disabled by default, and used only if set explicitly (--profile )" << Endl + << "Some commands do not require global options which required otherwise" + << colors.OldColor(); + stream << Endl << Endl + << colors.BoldColor() << "Description" << colors.OldColor() << ": " << Description << Endl << Endl + << colors.BoldColor() << "Subcommands" << colors.OldColor() << ":" << Endl; + RenderCommandsDescription(stream, colors); + stream << Endl; + PrintParentOptions(stream, config, colors); + config.Opts->SetCmdLineDescr(stream.Str()); +} + + } } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_admin.h b/ydb/public/lib/ydb_cli/commands/ydb_admin.h index dbdc75d8c61b..d544544ec924 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_admin.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_admin.h @@ -8,6 +8,8 @@ namespace NConsoleClient { class TCommandAdmin : public TClientCommandTree { public: TCommandAdmin(); +protected: + virtual void Config(TConfig& config) override; }; } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp b/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp index 78db85a9009d..fd4a24129cd2 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_cluster.cpp @@ -1,6 +1,7 @@ #include "ydb_cluster.h" #include +#include "ydb_dynamic_config.h" using namespace NKikimr; @@ -10,6 +11,7 @@ TCommandCluster::TCommandCluster() : TClientCommandTree("cluster", {}, "Cluster-wide administration") { AddCommand(std::make_unique()); + AddCommand(std::make_unique(std::nullopt, true)); } TCommandClusterBootstrap::TCommandClusterBootstrap() diff --git a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp index d6e2995b8e6b..ec97c95a657f 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.cpp @@ -22,16 +22,31 @@ TString WrapYaml(const TString& yaml) { return out.Str(); } -TCommandConfig::TCommandConfig() +TCommandConfig::TCommandConfig(std::optional overrideOnlyExplicitProfile, bool allowEmptyDatabase) : TClientCommandTree("config", {}, "Dynamic config") + , OverrideOnlyExplicitProfile(overrideOnlyExplicitProfile) { - AddCommand(std::make_unique()); - AddCommand(std::make_unique()); + AddCommand(std::make_unique(allowEmptyDatabase)); + AddCommand(std::make_unique(allowEmptyDatabase)); AddCommand(std::make_unique()); } -TCommandConfigFetch::TCommandConfigFetch() + +void TCommandConfig::PropagateFlags(const TCommandFlags& flags) { + TClientCommand::PropagateFlags(flags); + + if (OverrideOnlyExplicitProfile) { + OnlyExplicitProfile = *OverrideOnlyExplicitProfile; + } + + for (auto& [_, cmd] : SubCommands) { + cmd->PropagateFlags(TCommandFlags{.Dangerous = Dangerous, .OnlyExplicitProfile = OnlyExplicitProfile}); + } +} + +TCommandConfigFetch::TCommandConfigFetch(bool allowEmptyDatabase) : TYdbCommand("fetch", {"get", "dump"}, "Fetch main dynamic-config") + , AllowEmptyDatabase(allowEmptyDatabase) { } @@ -45,6 +60,7 @@ void TCommandConfigFetch::Config(TConfig& config) { .NoArgument().SetFlag(&StripMetadata); config.SetFreeArgsNum(0); + config.AllowEmptyDatabase = AllowEmptyDatabase; config.Opts->MutuallyExclusive("all", "strip-metadata"); config.Opts->MutuallyExclusive("output-directory", "strip-metadata"); } @@ -105,9 +121,10 @@ int TCommandConfigFetch::Run(TConfig& config) { return EXIT_SUCCESS; } -TCommandConfigReplace::TCommandConfigReplace() +TCommandConfigReplace::TCommandConfigReplace(bool allowEmptyDatabase) : TYdbCommand("replace", {}, "Replace dynamic config") , IgnoreCheck(false) + , AllowEmptyDatabase(allowEmptyDatabase) { } @@ -123,6 +140,7 @@ void TCommandConfigReplace::Config(TConfig& config) { .NoArgument().SetFlag(&AllowUnknownFields); config.Opts->AddLongOption("force", "Ignore metadata on config replacement") .NoArgument().SetFlag(&Force); + config.AllowEmptyDatabase = AllowEmptyDatabase; config.SetFreeArgsNum(0); } @@ -613,7 +631,6 @@ void TCommandConfigVolatileFetch::Config(TConfig& config) { config.Opts->AddLongOption("strip-metadata", "Strip metadata from config(s)") .NoArgument().SetFlag(&StripMetadata); config.SetFreeArgsNum(0); - config.Opts->MutuallyExclusive("output-directory", "strip-metadata"); } diff --git a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.h b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.h index 81e6748fa948..c5bade6f6235 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_dynamic_config.h @@ -9,12 +9,15 @@ namespace NYdb::NConsoleClient::NDynamicConfig { class TCommandConfig : public TClientCommandTree { public: - TCommandConfig(); + TCommandConfig(std::optional overrideOnlyExplicitProfile = std::nullopt, bool allowEmptyDatabase = false); + void PropagateFlags(const TCommandFlags& flags) override; +private: + std::optional OverrideOnlyExplicitProfile; }; class TCommandConfigReplace : public TYdbCommand { public: - TCommandConfigReplace(); + TCommandConfigReplace(bool allowEmptyDatabase); void Config(TConfig& config) override; void Parse(TConfig& config) override; int Run(TConfig& config) override; @@ -26,11 +29,12 @@ class TCommandConfigReplace : public TYdbCommand { bool AllowUnknownFields = false; TString DynamicConfig; TString Filename; + bool AllowEmptyDatabase = false; }; class TCommandConfigFetch : public TYdbCommand { public: - TCommandConfigFetch(); + TCommandConfigFetch(bool allowEmptyDatabase); void Config(TConfig&) override; void Parse(TConfig&) override; int Run(TConfig& config) override; @@ -39,6 +43,7 @@ class TCommandConfigFetch : public TYdbCommand { bool All = false; bool StripMetadata = false; TString OutDir; + bool AllowEmptyDatabase = false; }; class TCommandConfigResolve : public TYdbCommand { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp b/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp index 58ccceeac9d9..8072d25bdcc9 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_root_common.cpp @@ -36,7 +36,7 @@ TClientCommandRootCommon::TClientCommandRootCommon(const TString& name, const TC , Settings(settings) { ValidateSettings(); - AddCommand(std::make_unique()); + AddDangerousCommand(std::make_unique()); AddCommand(std::make_unique()); AddCommand(std::make_unique()); AddCommand(std::make_unique()); @@ -54,6 +54,7 @@ TClientCommandRootCommon::TClientCommandRootCommon(const TString& name, const TC AddCommand(std::make_unique()); AddCommand(std::make_unique()); AddCommand(std::make_unique()); + PropagateFlags(TCommandFlags{.Dangerous = false, .OnlyExplicitProfile = false}); } void TClientCommandRootCommon::ValidateSettings() { @@ -319,6 +320,7 @@ void TClientCommandRootCommon::Config(TConfig& config) { stream << " [options...] " << Endl << Endl << colors.BoldColor() << "Subcommands" << colors.OldColor() << ":" << Endl; RenderCommandsDescription(stream, colors); + stream << Endl << Endl << colors.BoldColor() << "Commands in " << colors.Red() << colors.BoldColor() << "admin" << colors.OldColor() << colors.BoldColor() << " subtree may treat global flags and profile differently, see corresponding help" << colors.OldColor() << Endl; opts.SetCmdLineDescr(stream.Str()); opts.GetLongOption("time").Hidden(); @@ -328,6 +330,11 @@ void TClientCommandRootCommon::Config(TConfig& config) { } void TClientCommandRootCommon::Parse(TConfig& config) { + TClientCommandRootBase::Parse(config); + config.VerbosityLevel = std::min(static_cast(VerbosityLevel), TConfig::EVerbosityLevel::DEBUG); +} + +void TClientCommandRootCommon::PostPrepare(TConfig& config) { if (ProfileFile.empty()) { config.ProfileFile = TStringBuilder() << HomeDir << '/' << Settings.YdbDir << "/config/config.yaml"; } else { @@ -339,12 +346,12 @@ void TClientCommandRootCommon::Parse(TConfig& config) { ProfileManager = CreateProfileManager(config.ProfileFile); ParseProfile(); - TClientCommandRootBase::Parse(config); ParseDatabase(config); ParseCaCerts(config); ParseIamEndpoint(config); - config.VerbosityLevel = std::min(static_cast(VerbosityLevel), TConfig::EVerbosityLevel::DEBUG); + ParseCredentials(config); + ParseAddress(config); } namespace { @@ -435,7 +442,7 @@ void TClientCommandRootCommon::ParseAddress(TConfig& config) { return; } // Priority 3. Active profile (if --profile option is not specified) - if (TryGetParamFromProfile("endpoint", ProfileManager->GetActiveProfile(), false, getAddress)) { + if (!config.OnlyExplicitProfile && TryGetParamFromProfile("endpoint", ProfileManager->GetActiveProfile(), false, getAddress)) { return; } } @@ -510,6 +517,7 @@ void TClientCommandRootCommon::ParseDatabase(TConfig& config) { config.ConnectionParams["database"].push_back({param, sourceText}); return false; }; + // Priority 1. Explicit --database option if (Database && getDatabase(Database, "explicit --database option", true)) { return; @@ -519,7 +527,7 @@ void TClientCommandRootCommon::ParseDatabase(TConfig& config) { return; } // Priority 3. Active profile (if --profile option is not specified) - if (TryGetParamFromProfile("database", ProfileManager->GetActiveProfile(), false, getDatabase)) { + if (!config.OnlyExplicitProfile && TryGetParamFromProfile("database", ProfileManager->GetActiveProfile(), false, getDatabase)) { return; } } @@ -574,19 +582,23 @@ void TClientCommandRootCommon::Validate(TConfig& config) { throw TMisuseException() << errors; } } + + // TODO: Maybe NeedToConnect doesn't always mean that we don't need to check endpoint and database + // TODO: Now we supplying only one error while it is possible to return all errors at once, + // maybe even erros from nested command's validate if (!config.NeedToConnect) { return; } - if (config.Address.empty()) { - throw TMisuseException() << "Missing required option 'endpoint'."; + if (config.Address.empty() && !config.AllowEmptyAddress) { + throw TMisuseException() << "Missing required option 'endpoint'." << (config.OnlyExplicitProfile ? " Profile ignored due to admin command use." : ""); } if (config.Database.empty() && config.AllowEmptyDatabase) { // just skip the Database check } else if (config.Database.empty()) { throw TMisuseException() - << "Missing required option 'database'."; + << "Missing required option 'database'." << (config.OnlyExplicitProfile ? " Profile ignored due to admin command use." : ""); } else if (!config.Database.StartsWith('/')) { throw TMisuseException() << "Path to a database \"" << config.Database << "\" is incorrect. It must be absolute and thus must begin with '/'."; diff --git a/ydb/public/lib/ydb_cli/commands/ydb_root_common.h b/ydb/public/lib/ydb_cli/commands/ydb_root_common.h index e87a587b2fb8..cfa571b41d04 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_root_common.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_root_common.h @@ -31,6 +31,7 @@ class TClientCommandRootCommon : public TClientCommandRootBase { public: TClientCommandRootCommon(const TString& name, const TClientSettings& settings); void Config(TConfig& config) override; + void PostPrepare(TConfig& config) override; void Parse(TConfig& config) override; void ParseAddress(TConfig& config) override; void ParseCredentials(TConfig& config) override; diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp index 75627fbe119a..a2cfa66dc4c2 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp @@ -43,10 +43,10 @@ void TCommandMakeDirectory::Config(TConfig& config) { void TCommandMakeDirectory::Parse(TConfig& config) { TClientCommand::Parse(config); - ParsePath(config, 0); } int TCommandMakeDirectory::Run(TConfig& config) { + ParsePath(config, 0); NScheme::TSchemeClient client(CreateDriver(config)); NStatusHelpers::ThrowOnErrorOrPrintIssues( client.MakeDirectory( @@ -78,10 +78,10 @@ void TCommandRemoveDirectory::Config(TConfig& config) { void TCommandRemoveDirectory::Parse(TConfig& config) { TClientCommand::Parse(config); - ParsePath(config, 0); } int TCommandRemoveDirectory::Run(TConfig& config) { + ParsePath(config, 0); TDriver driver = CreateDriver(config); NScheme::TSchemeClient schemeClient(driver); const auto settings = FillSettings(NScheme::TRemoveDirectorySettings()); @@ -245,10 +245,10 @@ void TCommandDescribe::Parse(TConfig& config) { TClientCommand::Parse(config); Database = config.Database; ParseOutputFormats(); - ParsePath(config, 0); } int TCommandDescribe::Run(TConfig& config) { + ParsePath(config, 0); TDriver driver = CreateDriver(config); NScheme::TSchemeClient client(driver); NScheme::TDescribePathResult result = client.DescribePath( @@ -1059,7 +1059,6 @@ void TCommandList::Config(TConfig& config) { void TCommandList::Parse(TConfig& config) { TClientCommand::Parse(config); - ParsePath(config, 0, true); if (AdvancedMode && FromNewLine) { // TODO: add "consider using --format shell" throw TMisuseException() << "Options -1 and -l are incompatible"; @@ -1067,6 +1066,7 @@ void TCommandList::Parse(TConfig& config) { } int TCommandList::Run(TConfig& config) { + ParsePath(config, 0, true); TDriver driver = CreateDriver(config); ISchemePrinter::TSettings settings = { Path, @@ -1129,7 +1129,6 @@ void TCommandPermissionGrant::Config(TConfig& config) { void TCommandPermissionGrant::Parse(TConfig& config) { TClientCommand::Parse(config); - ParsePath(config, 0); Subject = config.ParseResult->GetFreeArgs()[1]; if (Subject.empty()) { throw TMisuseException() << "Missing required argument "; @@ -1140,6 +1139,7 @@ void TCommandPermissionGrant::Parse(TConfig& config) { } int TCommandPermissionGrant::Run(TConfig& config) { + ParsePath(config, 0); NScheme::TSchemeClient client(CreateDriver(config)); NStatusHelpers::ThrowOnErrorOrPrintIssues( client.ModifyPermissions( @@ -1170,7 +1170,6 @@ void TCommandPermissionRevoke::Config(TConfig& config) { void TCommandPermissionRevoke::Parse(TConfig& config) { TClientCommand::Parse(config); - ParsePath(config, 0); Subject = config.ParseResult->GetFreeArgs()[1]; if (Subject.empty()) { throw TMisuseException() << "Missing required argument "; @@ -1181,6 +1180,7 @@ void TCommandPermissionRevoke::Parse(TConfig& config) { } int TCommandPermissionRevoke::Run(TConfig& config) { + ParsePath(config, 0); NScheme::TSchemeClient client(CreateDriver(config)); NStatusHelpers::ThrowOnErrorOrPrintIssues( client.ModifyPermissions( @@ -1211,7 +1211,6 @@ void TCommandPermissionSet::Config(TConfig& config) { void TCommandPermissionSet::Parse(TConfig& config) { TClientCommand::Parse(config); - ParsePath(config, 0); Subject = config.ParseResult->GetFreeArgs()[1]; if (Subject.empty()) { throw TMisuseException() << "Missing required argument "; @@ -1222,6 +1221,7 @@ void TCommandPermissionSet::Parse(TConfig& config) { } int TCommandPermissionSet::Run(TConfig& config) { + ParsePath(config, 0); NScheme::TSchemeClient client(CreateDriver(config)); NStatusHelpers::ThrowOnErrorOrPrintIssues( client.ModifyPermissions( @@ -1249,7 +1249,6 @@ void TCommandChangeOwner::Config(TConfig& config) { void TCommandChangeOwner::Parse(TConfig& config) { TClientCommand::Parse(config); - ParsePath(config, 0); Owner = config.ParseResult->GetFreeArgs()[1]; if (!Owner){ throw TMisuseException() << "Missing required argument "; @@ -1257,6 +1256,7 @@ void TCommandChangeOwner::Parse(TConfig& config) { } int TCommandChangeOwner::Run(TConfig& config) { + ParsePath(config, 0); NScheme::TSchemeClient client(CreateDriver(config)); NStatusHelpers::ThrowOnErrorOrPrintIssues( client.ModifyPermissions( @@ -1283,10 +1283,10 @@ void TCommandPermissionClear::Config(TConfig& config) { void TCommandPermissionClear::Parse(TConfig& config) { TClientCommand::Parse(config); - ParsePath(config, 0); } int TCommandPermissionClear::Run(TConfig& config) { + ParsePath(config, 0); NScheme::TSchemeClient client(CreateDriver(config)); NStatusHelpers::ThrowOnErrorOrPrintIssues( client.ModifyPermissions( @@ -1313,10 +1313,10 @@ void TCommandPermissionSetInheritance::Config(TConfig& config) { void TCommandPermissionSetInheritance::Parse(TConfig& config) { TClientCommand::Parse(config); - ParsePath(config, 0); } int TCommandPermissionSetInheritance::Run(TConfig& config) { + ParsePath(config, 0); NScheme::TSchemeClient client(CreateDriver(config)); NStatusHelpers::ThrowOnErrorOrPrintIssues( client.ModifyPermissions( @@ -1343,10 +1343,10 @@ void TCommandPermissionClearInheritance::Config(TConfig& config) { void TCommandPermissionClearInheritance::Parse(TConfig& config) { TClientCommand::Parse(config); - ParsePath(config, 0); } int TCommandPermissionClearInheritance::Run(TConfig& config) { + ParsePath(config, 0); NScheme::TSchemeClient client(CreateDriver(config)); NStatusHelpers::ThrowOnErrorOrPrintIssues( client.ModifyPermissions( @@ -1373,10 +1373,10 @@ void TCommandPermissionList::Config(TConfig& config) { void TCommandPermissionList::Parse(TConfig& config) { TClientCommand::Parse(config); - ParsePath(config, 0); } int TCommandPermissionList::Run(TConfig& config) { + ParsePath(config, 0); TDriver driver = CreateDriver(config); NScheme::TSchemeClient client(driver); NScheme::TDescribePathResult result = client.DescribePath( diff --git a/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp b/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp index 87f17b751a2c..5fb2cfd72c04 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_storage_config.cpp @@ -22,13 +22,26 @@ TString WrapYaml(const TString& yaml) { return out.Str(); } -TCommandStorageConfig::TCommandStorageConfig() +TCommandStorageConfig::TCommandStorageConfig(std::optional overrideOnlyExplicitProfile) : TClientCommandTree("storage", {}, "Storage config") + , OverrideOnlyExplicitProfile(overrideOnlyExplicitProfile) { AddCommand(std::make_unique()); AddCommand(std::make_unique()); } +void TCommandStorageConfig::PropagateFlags(const TCommandFlags& flags) { + TClientCommand::PropagateFlags(flags); + + if (OverrideOnlyExplicitProfile) { + OnlyExplicitProfile = *OverrideOnlyExplicitProfile; + } + + for (auto& [_, cmd] : SubCommands) { + cmd->PropagateFlags(TCommandFlags{.Dangerous = Dangerous, .OnlyExplicitProfile = OnlyExplicitProfile}); + } +} + TCommandStorageConfigFetch::TCommandStorageConfigFetch() : TYdbCommand("fetch", {}, "Fetch storage config") { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_storage_config.h b/ydb/public/lib/ydb_cli/commands/ydb_storage_config.h index a5c440a569ee..1092824e0e26 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_storage_config.h +++ b/ydb/public/lib/ydb_cli/commands/ydb_storage_config.h @@ -9,7 +9,10 @@ namespace NYdb::NConsoleClient::NStorageConfig { class TCommandStorageConfig : public TClientCommandTree { public: - TCommandStorageConfig(); + TCommandStorageConfig(std::optional overrideOnlyExplicitProfile = std::nullopt); + void PropagateFlags(const TCommandFlags& flags) override; +private: + std::optional OverrideOnlyExplicitProfile; }; class TCommandStorageConfigReplace : public TYdbCommand { diff --git a/ydb/public/lib/ydb_cli/commands/ydb_tools.cpp b/ydb/public/lib/ydb_cli/commands/ydb_tools.cpp index c24e8c639030..3821105587a2 100644 --- a/ydb/public/lib/ydb_cli/commands/ydb_tools.cpp +++ b/ydb/public/lib/ydb_cli/commands/ydb_tools.cpp @@ -86,10 +86,10 @@ void TCommandDump::Config(TConfig& config) { void TCommandDump::Parse(TConfig& config) { TClientCommand::Parse(config); - AdjustPath(config); } int TCommandDump::Run(TConfig& config) { + AdjustPath(config); NDump::TDumpSettings::EConsistencyLevel consistencyLevel; if (!TryFromString(ConsistencyLevel, consistencyLevel)) { throw yexception() << "Incorrect consistency level." diff --git a/ydb/public/lib/ydb_cli/common/command.cpp b/ydb/public/lib/ydb_cli/common/command.cpp index c4f8dde9e5f5..bdad74afb416 100644 --- a/ydb/public/lib/ydb_cli/common/command.cpp +++ b/ydb/public/lib/ydb_cli/common/command.cpp @@ -208,6 +208,7 @@ void TClientCommand::CheckForExecutableOptions(TConfig& config) { void TClientCommand::Config(TConfig& config) { config.Opts = &Opts; + config.OnlyExplicitProfile = OnlyExplicitProfile; TStringStream stream; NColorizer::TColors colors = NColorizer::AutoColors(Cout); stream << Endl << Endl @@ -251,9 +252,14 @@ void TClientCommand::Prepare(TConfig& config) { Parse(config); } +void TClientCommand::PostPrepare(TConfig& config) { + Y_UNUSED(config); +} + int TClientCommand::ValidateAndRun(TConfig& config) { config.Opts = &Opts; config.ParseResult = ParseResult.get(); + PostPrepare(config); Validate(config); return Run(config); } @@ -305,7 +311,7 @@ void TClientCommand::RenderOneCommandDescription( const NColorizer::TColors& colors, RenderEntryType type ) { - if (Hidden) { + if (Hidden && type != BEGIN) { return; } TString prefix; @@ -316,7 +322,7 @@ void TClientCommand::RenderOneCommandDescription( prefix = "└─ "; } TString line = prefix + Name; - stream << prefix << colors.BoldColor() << Name << colors.OldColor(); + stream << prefix << (Dangerous ? colors.Red() : "") << colors.BoldColor() << Name << colors.OldColor(); if (!Description.empty()) { int namePartLength = GetNumberOfUTF8Chars(line); if (namePartLength < DESCRIPTION_ALIGNMENT) @@ -339,6 +345,15 @@ void TClientCommand::RenderOneCommandDescription( void TClientCommand::Hide() { Hidden = true; + Visible = false; +} + +void TClientCommand::MarkDangerous() { + Dangerous = true; +} + +void TClientCommand::UseOnlyExplicitProfile() { + OnlyExplicitProfile = true; } TClientCommandTree::TClientCommandTree(const TString& name, const std::initializer_list& aliases, const TString& description) @@ -361,6 +376,12 @@ void TClientCommandTree::AddHiddenCommand(std::unique_ptr comman AddCommand(std::move(command)); } +void TClientCommandTree::AddDangerousCommand(std::unique_ptr command) { + command->MarkDangerous(); + command->UseOnlyExplicitProfile(); + AddCommand(std::move(command)); +} + void TClientCommandTree::Config(TConfig& config) { TClientCommand::Config(config); SetFreeArgs(config); diff --git a/ydb/public/lib/ydb_cli/common/command.h b/ydb/public/lib/ydb_cli/common/command.h index c1e474275be2..199e0db1ffde 100644 --- a/ydb/public/lib/ydb_cli/common/command.h +++ b/ydb/public/lib/ydb_cli/common/command.h @@ -17,6 +17,11 @@ namespace NYdb { namespace NConsoleClient { +struct TCommandFlags { + bool Dangerous = false; + bool OnlyExplicitProfile = false; +}; + class TClientCommand { public: static bool TIME_REQUESTS; // measure time of requests @@ -25,6 +30,9 @@ class TClientCommand { TVector Aliases; TString Description; bool Visible = true; + bool Hidden = false; + bool Dangerous = false; + bool OnlyExplicitProfile = false; const TClientCommand* Parent; NLastGetopt::TOpts Opts; TString Argument; @@ -137,6 +145,8 @@ class TClientCommand { bool NeedToCheckForUpdate = true; bool ForceVersionCheck = false; bool AllowEmptyDatabase = false; + bool AllowEmptyAddress = false; + bool OnlyExplicitProfile = false; TCredentialsGetter CredentialsGetter; @@ -307,7 +317,12 @@ class TClientCommand { virtual int Process(TConfig& config); virtual void Prepare(TConfig& config); + virtual void PostPrepare(TConfig& config); virtual int ValidateAndRun(TConfig& config); + virtual void PropagateFlags(const TCommandFlags& flags) { + Dangerous |= flags.Dangerous; + OnlyExplicitProfile |= flags.OnlyExplicitProfile; + } enum RenderEntryType { BEGIN, @@ -322,6 +337,8 @@ class TClientCommand { ); void Hide(); + void MarkDangerous(); + void UseOnlyExplicitProfile(); protected: virtual void Config(TConfig& config); @@ -342,7 +359,6 @@ class TClientCommand { void CheckForExecutableOptions(TConfig& config); constexpr static int DESCRIPTION_ALIGNMENT = 28; - bool Hidden = false; }; class TClientCommandTree : public TClientCommand { @@ -350,6 +366,7 @@ class TClientCommandTree : public TClientCommand { TClientCommandTree(const TString& name, const std::initializer_list& aliases = std::initializer_list(), const TString& description = TString()); void AddCommand(std::unique_ptr command); void AddHiddenCommand(std::unique_ptr command); + void AddDangerousCommand(std::unique_ptr command); virtual void Prepare(TConfig& config) override; void RenderCommandsDescription( TStringStream& stream, @@ -363,10 +380,15 @@ class TClientCommandTree : public TClientCommand { virtual void SaveParseResult(TConfig& config) override; virtual void Parse(TConfig& config) override; virtual int Run(TConfig& config) override; + virtual void PropagateFlags(const TCommandFlags& flags) override { + TClientCommand::PropagateFlags(flags); + for (auto& [_, cmd] : SubCommands) { + cmd->PropagateFlags(TCommandFlags{.Dangerous = Dangerous, .OnlyExplicitProfile = OnlyExplicitProfile}); + } + } TClientCommand* SelectedCommand; -private: bool HasOptionsToShow(); TMap> SubCommands; diff --git a/ydb/public/lib/ydb_cli/common/root.cpp b/ydb/public/lib/ydb_cli/common/root.cpp index 98ab7bc1ccd9..3ebbbd2686e7 100644 --- a/ydb/public/lib/ydb_cli/common/root.cpp +++ b/ydb/public/lib/ydb_cli/common/root.cpp @@ -37,8 +37,6 @@ void TClientCommandRootBase::SetCustomUsage(TConfig& config) { void TClientCommandRootBase::Parse(TConfig& config) { TClientCommandTree::Parse(config); - ParseCredentials(config); - ParseAddress(config); TClientCommand::TIME_REQUESTS = TimeRequests; TClientCommand::PROGRESS_REQUESTS = ProgressRequests; diff --git a/ydb/public/sdk/cpp/src/client/draft/ydb_dynamic_config.cpp b/ydb/public/sdk/cpp/src/client/draft/ydb_dynamic_config.cpp index ddcbcaed3850..f30bf96ddf1c 100644 --- a/ydb/public/sdk/cpp/src/client/draft/ydb_dynamic_config.cpp +++ b/ydb/public/sdk/cpp/src/client/draft/ydb_dynamic_config.cpp @@ -156,9 +156,9 @@ class TDynamicConfigClient::TImpl : public TClientImplCommon volatileConfigs; if (Ydb::DynamicConfig::GetConfigResult result; any && any->UnpackTo(&result)) { - clusterName = result.identity().cluster(); - version = result.identity().version(); - config = result.config(); + clusterName = result.identity(0).cluster(); + version = result.identity(0).version(); + config = result.config(0); for (const auto& config : result.volatile_configs()) { volatileConfigs.emplace(config.id(), config.config()); } diff --git a/ydb/tests/functional/audit/test_auditlog.py b/ydb/tests/functional/audit/test_auditlog.py index 166db3a50b2b..f4f8d4534cf4 100644 --- a/ydb/tests/functional/audit/test_auditlog.py +++ b/ydb/tests/functional/audit/test_auditlog.py @@ -434,7 +434,8 @@ def test_dynconfig(ydb_cluster, prepared_test_env, _client_session_pool_with_aut _client_session_pool_with_auth_root.retry_operation_sync(apply_config, config=config) print(capture_audit.captured, file=sys.stderr) - assert json.dumps(config) in capture_audit.captured + cfg = json.dumps(config) + assert cfg in capture_audit.captured @pytest.mark.parametrize('config_fixture', ["_bad_dynconfig", "_good_dynconfig"]) @@ -450,4 +451,5 @@ def test_broken_dynconfig(ydb_cluster, prepared_test_env, pool_fixture, config_f pass print(capture_audit.captured, file=sys.stderr) - assert json.dumps(config) in capture_audit.captured + cfg = json.dumps(config) + assert cfg in capture_audit.captured diff --git a/ydb/tests/functional/audit/ya.make b/ydb/tests/functional/audit/ya.make index e843ec0fd0be..8c49ebc0af28 100644 --- a/ydb/tests/functional/audit/ya.make +++ b/ydb/tests/functional/audit/ya.make @@ -2,6 +2,11 @@ PY3TEST() FORK_SUBTESTS() FORK_TEST_FILES() +# It is necessary to run all tests +# in separate chunks because our +# audit log capture method is unreliable +# and therefore some tests may affect neighbouring ones +SPLIT_FACTOR(100) SIZE(MEDIUM) ENV(YDB_USE_IN_MEMORY_PDISKS=true)