Topics: add partition-level metrics (25-3-1) (#26173)

qyryq · web-flow · commit 2bf0f3dfe4e1 · 2025-10-03T14:54:16.000+03:00
diff --git a/ydb/core/persqueue/pqtablet/partition/partition_init.cpp b/ydb/core/persqueue/pqtablet/partition/partition_init.cpp
@@ -2,8 +2,9 @@
 #include "offload_actor.h"
 #include "partition.h"
 #include "partition_compactification.h"
-#include <ydb/core/persqueue/pqtablet/common/logging.h>
 #include "partition_util.h"
+#include <ydb/core/persqueue/pqtablet/common/logging.h>
+#include <ydb/core/persqueue/pqtablet/common/constants.h>
 
 #include <memory>
 
@@ -1101,7 +1102,7 @@ void TPartition::Initialize(const TActorContext& ctx) {
 }
 
 bool TPartition::DetailedMetricsAreEnabled() const {
-    return AppData()->FeatureFlags.GetEnableMetricsLevel() && (Config.HasMetricsLevel() && Config.GetMetricsLevel() == Ydb::MetricsLevel::Detailed);
+    return AppData()->FeatureFlags.GetEnableMetricsLevel() && (Config.HasMetricsLevel() && Config.GetMetricsLevel() == METRICS_LEVEL_DETAILED);
 }
 
 void TPartition::SetupTopicCounters(const TActorContext& ctx) {
diff --git a/ydb/core/persqueue/pqtablet/partition/user_info.cpp b/ydb/core/persqueue/pqtablet/partition/user_info.cpp
@@ -1,4 +1,5 @@
 #include "user_info.h"
+#include <ydb/core/persqueue/pqtablet/common/constants.h>
 
 namespace NKikimr {
 namespace NPQ {
@@ -438,7 +439,7 @@ void TUsersInfoStorage::ResetDetailedMetrics() {
 }
 
 bool TUsersInfoStorage::DetailedMetricsAreEnabled() const {
-    return AppData()->FeatureFlags.GetEnableMetricsLevel() && (Config.HasMetricsLevel() && Config.GetMetricsLevel() == Ydb::MetricsLevel::Detailed);
+    return AppData()->FeatureFlags.GetEnableMetricsLevel() && (Config.HasMetricsLevel() && Config.GetMetricsLevel() == METRICS_LEVEL_DETAILED);
 }
 
 const TUserInfo* TUsersInfoStorage::GetIfExists(const TString& user) const {
diff --git a/ydb/core/persqueue/public/constants.h b/ydb/core/persqueue/public/constants.h
@@ -7,4 +7,9 @@ namespace NKikimr::NPQ {
 static const TString CLIENTID_COMPACTION_CONSUMER = "__ydb_compaction_consumer";
 static const TString CLIENTID_WITHOUT_CONSUMER = "$without_consumer";
 
+static constexpr ui32 METRICS_LEVEL_DISABLED = 0;
+static constexpr ui32 METRICS_LEVEL_DATABASE = 1;
+static constexpr ui32 METRICS_LEVEL_OBJECT = 2;
+static constexpr ui32 METRICS_LEVEL_DETAILED = 3;
+
 }
diff --git a/ydb/core/persqueue/ut/counters_ut.cpp b/ydb/core/persqueue/ut/counters_ut.cpp
@@ -5,6 +5,7 @@
 #include <ydb/core/mon/mon.h>
 #include <ydb/core/persqueue/ut/common/pq_ut_common.h>
 #include <ydb/core/persqueue/public/counters/percentile_counter.h>
+#include <ydb/core/persqueue/pqtablet/common/constants.h>
 #include <ydb/core/persqueue/pqtablet/partition/partition.h>
 #include <ydb/core/sys_view/service/sysview_service.h>
 #include <ydb/core/testlib/fake_scheme_shard.h>
@@ -126,7 +127,7 @@ void PartitionLevelCounters(bool featureFlagEnabled, bool firstClassCitizen, TSt
 
     tc.Runtime->GetAppData(0).FeatureFlags.SetEnableMetricsLevel(featureFlagEnabled);
 
-    PQTabletPrepare({ .metricsLevel = 1 }, {}, tc);
+    PQTabletPrepare({ .metricsLevel = METRICS_LEVEL_OBJECT }, {}, tc);
     CmdWrite(0, "sourceid0", TestData(), tc, false, {}, true);
     CmdWrite(0, "sourceid1", TestData(), tc, false);
     CmdWrite(0, "sourceid2", TestData(), tc, false);
@@ -166,7 +167,7 @@ void PartitionLevelCounters(bool featureFlagEnabled, bool firstClassCitizen, TSt
     {
         // Turn on per partition counters, check counters.
 
-        PQTabletPrepare({ .metricsLevel = 2 }, {}, tc);
+        PQTabletPrepare({ .metricsLevel = METRICS_LEVEL_DETAILED }, {}, tc);
 
         // partition, sourceId, data, text
         CmdWrite({ .Partition = 0, .SourceId = "sourceid3", .Data = TestData(), .TestContext = tc, .Error = false });
@@ -237,7 +238,7 @@ void PartitionLevelCounters(bool featureFlagEnabled, bool firstClassCitizen, TSt
     {
         // Disable per partition counters, the counters should be empty.
 
-        PQTabletPrepare({ .metricsLevel = 1 }, {}, tc);
+        PQTabletPrepare({ .metricsLevel = METRICS_LEVEL_OBJECT }, {}, tc);
         TString counters = getCountersHtml();
         TString referenceCounters = NResource::Find(TStringBuilder() << referenceDir << "_turned_off.html");
         counters = zeroUnreliableValues(counters) + (featureFlagEnabled ? "\n" : "");
diff --git a/ydb/core/protos/pqconfig.proto b/ydb/core/protos/pqconfig.proto
@@ -1,7 +1,6 @@
 import "ydb/public/api/protos/draft/persqueue_error_codes.proto";
 
 import "ydb/public/api/protos/annotations/sensitive.proto";
-import "ydb/public/api/protos/ydb_common.proto";
 
 import "ydb/core/protos/base.proto";
 import "ydb/core/protos/msgbus_kv.proto";
@@ -447,7 +446,7 @@ message TPQTabletConfig {
 
     optional bool EnableCompactification = 39 [default = false];
 
-    optional Ydb.MetricsLevel MetricsLevel = 40;
+    optional uint32 MetricsLevel = 40;
 
     // The ID of the Monitoring project that will be used as the value of `project` label for detailed metrics level.
     optional string MonitoringProjectId = 41;
diff --git a/ydb/core/ydb_convert/topic_description.cpp b/ydb/core/ydb_convert/topic_description.cpp
@@ -43,7 +43,7 @@ bool FillConsumer(Ydb::Topic::Consumer& out, const NKikimrPQ::TPQTabletConfig_TC
 bool FillTopicDescription(Ydb::Topic::DescribeTopicResult& out, const NKikimrSchemeOp::TPersQueueGroupDescription& inDesc,
     const NKikimrSchemeOp::TDirEntry& inDirEntry, const TMaybe<TString>& cdcName,
     Ydb::StatusIds_StatusCode& status, TString& error) {
-    
+
     const NKikimrPQ::TPQConfig pqConfig = AppData()->PQConfig;
 
     Ydb::Scheme::Entry *selfEntry = out.mutable_self();
@@ -160,6 +160,10 @@ bool FillTopicDescription(Ydb::Topic::DescribeTopicResult& out, const NKikimrSch
         }
     }
 
+    if (config.HasMetricsLevel()) {
+        out.set_metrics_level(config.GetMetricsLevel());
+    }
+
     for (const auto& consumer : config.GetConsumers()) {
         if (!FillConsumer(*out.add_consumers(), consumer, status, error)) {
             return false;
diff --git a/ydb/public/api/protos/ydb_common.proto b/ydb/public/api/protos/ydb_common.proto
@@ -27,9 +27,3 @@ message VirtualTimestamp {
     uint64 plan_step = 1;
     uint64 tx_id = 2;
 }
-
-enum MetricsLevel {
-    Database = 0;
-    Object = 1;
-    Detailed = 2;
-}
diff --git a/ydb/public/api/protos/ydb_persqueue_v1.proto b/ydb/public/api/protos/ydb_persqueue_v1.proto
@@ -1184,6 +1184,9 @@ message TopicSettings {
     }
     // remote mirror rule for this topic.
     RemoteMirrorRule remote_mirror_rule = 11;
+
+    // Set or reset metrics level.
+    optional uint32 metrics_level = 16;
 }
 
 message AutoPartitioningSettings {
diff --git a/ydb/public/api/protos/ydb_topic.proto b/ydb/public/api/protos/ydb_topic.proto
@@ -7,6 +7,7 @@ import "ydb/public/api/protos/annotations/sensitive.proto";
 import "ydb/public/api/protos/annotations/validation.proto";
 
 import "google/protobuf/duration.proto";
+import "google/protobuf/empty.proto";
 import "google/protobuf/timestamp.proto";
 
 package Ydb.Topic;
@@ -992,6 +993,9 @@ message CreateTopicRequest {
 
     // Metering mode for the topic in a serverless database.
     MeteringMode metering_mode = 12;
+
+    // Metrics level. If the level is unset, use database setting.
+    optional uint32 metrics_level = 13;
 }
 
 // Create topic response sent from server to client.
@@ -1125,6 +1129,9 @@ message DescribeTopicResult {
         // How much bytes were written statistics.
         MultipleWindowsStat bytes_written = 4;
     }
+
+    // Metrics level.
+    optional uint32 metrics_level = 16;
 }
 
 // Describe partition request sent from client to server.
@@ -1252,7 +1259,6 @@ message PartitionStats {
     int32 partition_node_id = 8 [deprecated=true]; //Use PartitionLocation
 }
 
-
 // Update existing topic request sent from client to server.
 message AlterTopicRequest {
     Ydb.Operations.OperationParams operation_params = 1;
@@ -1297,6 +1303,12 @@ message AlterTopicRequest {
 
     // Set metering mode for topic in serverless database.
     MeteringMode set_metering_mode = 14;
+
+    // Set or reset metrics level.
+    oneof metrics_level {
+        uint32 set_metrics_level = 15;
+        google.protobuf.Empty reset_metrics_level = 16;
+    }
 }
 
 // Update topic response sent from server to client.
diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_topic.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_topic.cpp
@@ -35,6 +35,15 @@ namespace NYdb::NConsoleClient {
             std::pair<TString, NYdb::NTopic::ECodec>("zstd", NYdb::NTopic::ECodec::ZSTD),
         };
 
+        TVector<ui32> ExistingMetricsLevels = {2, 3};
+
+        THashMap<ui32, TString> MetricsLevelsDescriptions = {
+            {0, "No metrics."},
+            {1, "Database level metrics."},
+            {2, "Database and topic level metrics."},
+            {3, "Database, topic, and partition level metrics."},
+        };
+
         THashMap<TString, NTopic::EMeteringMode> ExistingMeteringModes = {
             std::pair<TString, NTopic::EMeteringMode>("request-units", NTopic::EMeteringMode::RequestUnits),
             std::pair<TString, NTopic::EMeteringMode>("reserved-capacity", NTopic::EMeteringMode::ReservedCapacity),
@@ -159,6 +168,29 @@ namespace NYdb::NConsoleClient {
         AllowedCodecs_ = supportedCodecs;
     }
 
+    void TCommandWithMetricsLevel::AddMetricsLevels(TClientCommand::TConfig& config) {
+        TStringStream description;
+        description << "Available metrics levels: ";
+        NColorizer::TColors colors = NColorizer::AutoColors(Cout);
+        for (const auto& level: ExistingMetricsLevels) {
+            auto findResult = MetricsLevelsDescriptions.find(level);
+            Y_ABORT_UNLESS(findResult != MetricsLevelsDescriptions.end(),
+                     "Couldn't find description for %s metrics level", (TStringBuilder() << level).c_str());
+            description << "\n  " << colors.BoldColor() << level << colors.OldColor()
+                        << "\n    " << findResult->second;
+            if (level == 2 /* database and topic level metrics */) {
+                description << colors.CyanColor() << " (default)" << colors.OldColor();
+            }
+        }
+        config.Opts->AddLongOption("metrics-level", description.Str())
+            .Optional()
+            .StoreResult(&MetricsLevel_);
+    }
+
+    TMaybe<NTopic::EMetricsLevel> TCommandWithMetricsLevel::GetMetricsLevel() const {
+        return MetricsLevel_;
+    }
+
     void TCommandWithSupportedCodecs::ParseCodecs() {
         TVector<NTopic::ECodec> parsedCodecs;
         TVector<TString> split = SplitString(SupportedCodecsStr_, ",");
@@ -323,6 +355,7 @@ namespace NYdb::NConsoleClient {
         SetFreeArgTitle(0, "<topic-path>", "Topic path");
         AddAllowedCodecs(config, AllowedCodecs);
         AddAllowedMeteringModes(config);
+        AddMetricsLevels(config);
 
         config.Opts->AddLongOption("auto-partitioning-max-partitions-count", "Maximum number of partitions for topic")
             .Optional()
@@ -379,6 +412,10 @@ namespace NYdb::NConsoleClient {
             settings.AddAttribute("_partitions_per_tablet", ToString(*PartitionsPerTablet_));
         }
 
+        if (auto level = GetMetricsLevel(); level.Defined()) {
+            settings.MetricsLevel(*level);
+        }
+
         auto status = topicClient.CreateTopic(TopicName, settings).GetValueSync();
         NStatusHelpers::ThrowOnErrorOrPrintIssues(status);
         return EXIT_SUCCESS;
@@ -406,6 +443,7 @@ namespace NYdb::NConsoleClient {
         SetFreeArgTitle(0, "<topic-path>", "Topic path");
         AddAllowedCodecs(config, AllowedCodecs);
         AddAllowedMeteringModes(config);
+        AddMetricsLevels(config);
 
         config.Opts->AddLongOption("auto-partitioning-max-partitions-count", "Maximum number of partitions for topic")
             .Optional()
@@ -476,6 +514,10 @@ namespace NYdb::NConsoleClient {
             settings.SetRetentionStorageMb(*RetentionStorageMb_);
         }
 
+        if (MetricsLevel_.Defined()) {
+            settings.SetMetricsLevel(*MetricsLevel_);
+        }
+
         return settings;
     }
 
diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_topic.h b/ydb/public/lib/ydb_cli/commands/ydb_service_topic.h
@@ -38,6 +38,15 @@ namespace NYdb::NConsoleClient {
         NTopic::EMeteringMode MeteringMode_ = NTopic::EMeteringMode::Unspecified;
     };
 
+    class TCommandWithMetricsLevel {
+    protected:
+        void AddMetricsLevels(TClientCommand::TConfig& config);
+        TMaybe<NTopic::EMetricsLevel> GetMetricsLevel() const;
+
+    private:
+        NTopic::EMetricsLevel MetricsLevel_;
+    };
+
     class TCommandWithAutoPartitioning {
     protected:
         void AddAutoPartitioning(TClientCommand::TConfig& config, bool withDefault);
@@ -61,7 +70,12 @@ namespace NYdb::NConsoleClient {
         TCommandTopic();
     };
 
-    class TCommandTopicCreate: public TYdbCommand, public TCommandWithTopicName, public TCommandWithSupportedCodecs, public TCommandWithMeteringMode, public TCommandWithAutoPartitioning {
+    class TCommandTopicCreate: public TYdbCommand,
+                               public TCommandWithTopicName,
+                               public TCommandWithSupportedCodecs,
+                               public TCommandWithMeteringMode,
+                               public TCommandWithAutoPartitioning,
+                               public TCommandWithMetricsLevel {
     public:
         TCommandTopicCreate();
         void Config(TConfig& config) override;
@@ -75,9 +89,15 @@ namespace NYdb::NConsoleClient {
         TMaybe<ui32> MaxActivePartitions_;
         ui32 PartitionWriteSpeedKbps_;
         TMaybe<ui32> PartitionsPerTablet_;
+        TMaybe<NTopic::EMetricsLevel> MetricsLevel_;
     };
 
-    class TCommandTopicAlter: public TYdbCommand, public TCommandWithTopicName, public TCommandWithSupportedCodecs, public TCommandWithMeteringMode, public TCommandWithAutoPartitioning {
+    class TCommandTopicAlter: public TYdbCommand,
+                              public TCommandWithTopicName,
+                              public TCommandWithSupportedCodecs,
+                              public TCommandWithMeteringMode,
+                              public TCommandWithAutoPartitioning,
+                              public TCommandWithMetricsLevel {
     public:
         TCommandTopicAlter();
         void Config(TConfig& config) override;
@@ -92,6 +112,7 @@ namespace NYdb::NConsoleClient {
 
 
         TMaybe<ui32> PartitionWriteSpeedKbps_;
+        TMaybe<NTopic::EMetricsLevel> MetricsLevel_;
 
         NYdb::NTopic::TAlterTopicSettings PrepareAlterSettings(NYdb::NTopic::TDescribeTopicResult& describeResult);
     };
diff --git a/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/control_plane.h b/ydb/public/sdk/cpp/include/ydb-cpp-sdk/client/topic/control_plane.h
@@ -36,6 +36,13 @@ enum class EAutoPartitioningStrategy: uint32_t {
     Paused = 4,
 };
 
+// 0 - unspecified
+// 1 - disabeld
+// 2 - database level metrics
+// 3 - object level metrics
+// 4 - detailed metrics
+using EMetricsLevel = uint32_t;
+
 class TConsumer {
 public:
     TConsumer(const Ydb::Topic::Consumer&);
@@ -307,6 +314,8 @@ class TTopicDescription {
 
     const TTopicStats& GetTopicStats() const;
 
+    std::optional<EMetricsLevel> GetMetricsLevel() const;
+
     void SerializeTo(Ydb::Topic::CreateTopicRequest& request) const;
 private:
 
@@ -330,6 +339,7 @@ class TTopicDescription {
     NScheme::TVirtualTimestamp CreationTimestamp_;
     std::vector<NScheme::TPermissions> Permissions_;
     std::vector<NScheme::TPermissions> EffectivePermissions_;
+    std::optional<EMetricsLevel> MetricsLevel_;
 };
 
 class TConsumerDescription {
@@ -557,6 +567,8 @@ struct TCreateTopicSettings : public TOperationRequestSettings<TCreateTopicSetti
 
     FLUENT_SETTING(TAttributes, Attributes);
 
+    FLUENT_SETTING_OPTIONAL(EMetricsLevel, MetricsLevel);
+
     TCreateTopicSettings& SetSupportedCodecs(std::vector<ECodec>&& codecs) {
         SupportedCodecs_ = std::move(codecs);
         return *this;
@@ -729,7 +741,20 @@ struct TAlterTopicSettings : public TOperationRequestSettings<TAlterTopicSetting
         return *this;
     }
 
+    TAlterTopicSettings& SetMetricsLevel(EMetricsLevel level) {
+        MetricsLevel_ = level;
+        return *this;
+    }
+    TAlterTopicSettings& ResetMetricsLevel() {
+        MetricsLevel_ = true;
+        return *this;
+    }
+
     std::optional<TAlterPartitioningSettings> AlterPartitioningSettings_;
+    std::variant<
+        bool,         // Reset
+        EMetricsLevel // Set
+    > MetricsLevel_;
 };
 
 inline TPartitioningSettingsBuilder TCreateTopicSettings::BeginConfigurePartitioningSettings() {
diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/topic.cpp b/ydb/public/sdk/cpp/src/client/topic/impl/topic.cpp
diff --git a/ydb/public/sdk/cpp/src/client/topic/impl/topic_impl.h b/ydb/public/sdk/cpp/src/client/topic/impl/topic_impl.h
diff --git a/ydb/public/sdk/cpp/tests/integration/topic/describe_topic.cpp b/ydb/public/sdk/cpp/tests/integration/topic/describe_topic.cpp
diff --git a/ydb/public/sdk/cpp/tests/integration/topic/utils/describe.cpp b/ydb/public/sdk/cpp/tests/integration/topic/utils/describe.cpp
diff --git a/ydb/services/lib/actors/pq_schema_actor.cpp b/ydb/services/lib/actors/pq_schema_actor.cpp

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`#include "user_info.h"`
	`2`	`+#include <ydb/core/persqueue/pqtablet/common/constants.h>`
`2`	`3`
`3`	`4`	`namespace NKikimr {`
`4`	`5`	`namespace NPQ {`
`@@ -438,7 +439,7 @@ void TUsersInfoStorage::ResetDetailedMetrics() {`
`438`	`439`	`}`
`439`	`440`
`440`	`441`	`bool TUsersInfoStorage::DetailedMetricsAreEnabled() const {`
`441`		`- return AppData()->FeatureFlags.GetEnableMetricsLevel() && (Config.HasMetricsLevel() && Config.GetMetricsLevel() == Ydb::MetricsLevel::Detailed);`
	`442`	`+ return AppData()->FeatureFlags.GetEnableMetricsLevel() && (Config.HasMetricsLevel() && Config.GetMetricsLevel() == METRICS_LEVEL_DETAILED);`
`442`	`443`	`}`
`443`	`444`
`444`	`445`	`const TUserInfo* TUsersInfoStorage::GetIfExists(const TString& user) const {`
Original file line number	Diff line number	Diff line change
`@@ -7,4 +7,9 @@ namespace NKikimr::NPQ {`
`7`	`7`	`static const TString CLIENTID_COMPACTION_CONSUMER = "__ydb_compaction_consumer";`
`8`	`8`	`static const TString CLIENTID_WITHOUT_CONSUMER = "$without_consumer";`
`9`	`9`
	`10`	`+static constexpr ui32 METRICS_LEVEL_DISABLED = 0;`
	`11`	`+static constexpr ui32 METRICS_LEVEL_DATABASE = 1;`
	`12`	`+static constexpr ui32 METRICS_LEVEL_OBJECT = 2;`
	`13`	`+static constexpr ui32 METRICS_LEVEL_DETAILED = 3;`
	`14`	`+`
`10`	`15`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1184,6 +1184,9 @@ message TopicSettings {`
`1184`	`1184`	`}`
`1185`	`1185`	`// remote mirror rule for this topic.`
`1186`	`1186`	`RemoteMirrorRule remote_mirror_rule = 11;`
	`1187`	`+`
	`1188`	`+ // Set or reset metrics level.`
	`1189`	`+ optional uint32 metrics_level = 16;`
`1187`	`1190`	`}`
`1188`	`1191`
`1189`	`1192`	`message AutoPartitioningSettings {`