Add logsdb telemetry (#115994)

dnhatn · web-flow · commit 6d4e11d6bc47 · 2024-11-01T08:25:40.000-07:00
This PR adds telemetry for logsdb. However, this change only tracks the 
count of indices using logsdb and those that use synthetic source. 
Additional stats, such as shard, indexing, and search stats, will be
added in a follow-up, as they require reaching out to data nodes.
diff --git a/docs/changelog/115994.yaml b/docs/changelog/115994.yaml
@@ -0,0 +1,5 @@
+pr: 115994
+summary: Add logsdb telemetry
+area: Logs
+type: enhancement
+issues: []
diff --git a/docs/reference/rest-api/info.asciidoc b/docs/reference/rest-api/info.asciidoc
@@ -172,6 +172,10 @@ Example response:
       "universal_profiling": {
          "available": true,
          "enabled": true
+      },
+      "logsdb": {
+        "available": true,
+        "enabled": false
       }
    },
    "tagline" : "You know, for X"
diff --git a/docs/reference/rest-api/usage.asciidoc b/docs/reference/rest-api/usage.asciidoc
@@ -518,6 +518,12 @@ GET /_xpack/usage
   "universal_profiling" : {
     "available" : true,
     "enabled" : true
+  },
+  "logsdb": {
+    "available": true,
+    "enabled": false,
+    "indices_count": 0,
+    "indices_with_synthetic_source": 0
   }
 }
 ------------------------------------------------------------
diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java
@@ -186,6 +186,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion CPU_STAT_STRING_PARSING = def(8_781_00_0);
     public static final TransportVersion QUERY_RULES_RETRIEVER = def(8_782_00_0);
     public static final TransportVersion ESQL_CCS_EXEC_INFO_WITH_FAILURES = def(8_783_00_0);
+    public static final TransportVersion LOGSDB_TELEMETRY = def(8_784_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
@@ -25,6 +25,7 @@
 import org.elasticsearch.xpack.core.aggregatemetric.AggregateMetricFeatureSetUsage;
 import org.elasticsearch.xpack.core.analytics.AnalyticsFeatureSetUsage;
 import org.elasticsearch.xpack.core.application.EnterpriseSearchFeatureSetUsage;
+import org.elasticsearch.xpack.core.application.LogsDBFeatureSetUsage;
 import org.elasticsearch.xpack.core.application.ProfilingUsage;
 import org.elasticsearch.xpack.core.archive.ArchiveFeatureSetUsage;
 import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
@@ -305,7 +306,8 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
                 PersistentTaskParams.class,
                 SecurityMigrationTaskParams.TASK_NAME,
                 SecurityMigrationTaskParams::new
-            )
+            ),
+            new NamedWriteableRegistry.Entry(XPackFeatureUsage.class, XPackField.LOGSDB, LogsDBFeatureSetUsage::new)
         ).filter(Objects::nonNull).toList();
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackFeatures.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackFeatures.java
@@ -20,12 +20,14 @@
  * Provides the XPack features that this version of the code supports
  */
 public class XPackFeatures implements FeatureSpecification {
+    public static final NodeFeature LOGSDB_TELEMETRY = new NodeFeature("logsdb_telemetry");
 
     @Override
     public Set<NodeFeature> getFeatures() {
         return Set.of(
             NodesDataTiersUsageTransportAction.LOCALLY_PRECALCULATED_STATS_FEATURE, // Added in 8.12
-            License.INDEPENDENT_TRIAL_VERSION_FEATURE // 8.14.0
+            License.INDEPENDENT_TRIAL_VERSION_FEATURE, // 8.14.0
+            LOGSDB_TELEMETRY
         );
     }
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java
@@ -90,6 +90,7 @@ public final class XPackField {
     public static final String ENTERPRISE_GEOIP_DOWNLOADER = "enterprise_geoip_downloader";
     /** Name for Universal Profiling. */
     public static final String UNIVERSAL_PROFILING = "universal_profiling";
+    public static final String LOGSDB = "logsdb";
 
     private XPackField() {}
 
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java
@@ -48,6 +48,7 @@ public class XPackInfoFeatureAction {
     public static final ActionType<XPackInfoFeatureResponse> ARCHIVE = xpackInfoFeatureAction(XPackField.ARCHIVE);
     public static final ActionType<XPackInfoFeatureResponse> ENTERPRISE_SEARCH = xpackInfoFeatureAction(XPackField.ENTERPRISE_SEARCH);
     public static final ActionType<XPackInfoFeatureResponse> UNIVERSAL_PROFILING = xpackInfoFeatureAction(XPackField.UNIVERSAL_PROFILING);
+    public static final ActionType<XPackInfoFeatureResponse> LOGSDB = xpackInfoFeatureAction(XPackField.LOGSDB);
 
     public static final List<ActionType<XPackInfoFeatureResponse>> ALL = List.of(
         SECURITY,
@@ -75,7 +76,8 @@ public class XPackInfoFeatureAction {
         AGGREGATE_METRIC,
         ARCHIVE,
         ENTERPRISE_SEARCH,
-        UNIVERSAL_PROFILING
+        UNIVERSAL_PROFILING,
+        LOGSDB
     );
 
     public static ActionType<XPackInfoFeatureResponse> xpackInfoFeatureAction(String suffix) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java
@@ -58,6 +58,7 @@ private XPackUsageFeatureAction() {/* no instances */}
     public static final ActionType<XPackUsageFeatureResponse> REMOTE_CLUSTERS = xpackUsageFeatureAction(XPackField.REMOTE_CLUSTERS);
     public static final ActionType<XPackUsageFeatureResponse> ENTERPRISE_SEARCH = xpackUsageFeatureAction(XPackField.ENTERPRISE_SEARCH);
     public static final ActionType<XPackUsageFeatureResponse> UNIVERSAL_PROFILING = xpackUsageFeatureAction(XPackField.UNIVERSAL_PROFILING);
+    public static final ActionType<XPackUsageFeatureResponse> LOGSDB = xpackUsageFeatureAction(XPackField.LOGSDB);
 
     static final List<ActionType<XPackUsageFeatureResponse>> ALL = List.of(
         AGGREGATE_METRIC,
@@ -88,7 +89,8 @@ private XPackUsageFeatureAction() {/* no instances */}
         HEALTH,
         REMOTE_CLUSTERS,
         ENTERPRISE_SEARCH,
-        UNIVERSAL_PROFILING
+        UNIVERSAL_PROFILING,
+        LOGSDB
     );
 
     public static ActionType<XPackUsageFeatureResponse> xpackUsageFeatureAction(String suffix) {
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/application/LogsDBFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/application/LogsDBFeatureSetUsage.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.core.application;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.XPackFeatureUsage;
+import org.elasticsearch.xpack.core.XPackField;
+
+import java.io.IOException;
+import java.util.Objects;
+
+public final class LogsDBFeatureSetUsage extends XPackFeatureUsage {
+    private final int indicesCount;
+    private final int indicesWithSyntheticSource;
+
+    public LogsDBFeatureSetUsage(StreamInput input) throws IOException {
+        super(input);
+        indicesCount = input.readVInt();
+        indicesWithSyntheticSource = input.readVInt();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeVInt(indicesCount);
+        out.writeVInt(indicesWithSyntheticSource);
+    }
+
+    public LogsDBFeatureSetUsage(boolean available, boolean enabled, int indicesCount, int indicesWithSyntheticSource) {
+        super(XPackField.LOGSDB, available, enabled);
+        this.indicesCount = indicesCount;
+        this.indicesWithSyntheticSource = indicesWithSyntheticSource;
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.LOGSDB_TELEMETRY;
+    }
+
+    @Override
+    protected void innerXContent(XContentBuilder builder, Params params) throws IOException {
+        super.innerXContent(builder, params);
+        builder.field("indices_count", indicesCount);
+        builder.field("indices_with_synthetic_source", indicesWithSyntheticSource);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(available, enabled, indicesCount, indicesWithSyntheticSource);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        LogsDBFeatureSetUsage other = (LogsDBFeatureSetUsage) obj;
+        return Objects.equals(available, other.available)
+            && Objects.equals(enabled, other.enabled)
+            && Objects.equals(indicesCount, other.indicesCount)
+            && Objects.equals(indicesWithSyntheticSource, other.indicesWithSyntheticSource);
+    }
+}
diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBInfoTransportAction.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBInfoTransportAction.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.logsdb;
+
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackField;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction;
+
+public class LogsDBInfoTransportAction extends XPackInfoFeatureTransportAction {
+
+    private final ClusterService clusterService;
+
+    @Inject
+    public LogsDBInfoTransportAction(TransportService transportService, ClusterService clusterService, ActionFilters actionFilters) {
+        super(XPackInfoFeatureAction.LOGSDB.name(), transportService, actionFilters);
+        this.clusterService = clusterService;
+    }
+
+    @Override
+    public String name() {
+        return XPackField.LOGSDB;
+    }
+
+    @Override
+    public boolean available() {
+        return true;
+    }
+
+    @Override
+    public boolean enabled() {
+        return LogsDBPlugin.CLUSTER_LOGSDB_ENABLED.get(clusterService.getSettings());
+    }
+}
diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java
@@ -7,23 +7,34 @@
 
 package org.elasticsearch.xpack.logsdb;
 
+import org.elasticsearch.action.ActionRequest;
+import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexSettingProvider;
+import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.xpack.core.XPackPlugin;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 
-import static org.elasticsearch.xpack.logsdb.LogsdbIndexModeSettingsProvider.CLUSTER_LOGSDB_ENABLED;
 import static org.elasticsearch.xpack.logsdb.SyntheticSourceLicenseService.FALLBACK_SETTING;
 
-public class LogsDBPlugin extends Plugin {
+public class LogsDBPlugin extends Plugin implements ActionPlugin {
 
     private final Settings settings;
     private final SyntheticSourceLicenseService licenseService;
+    public static final Setting<Boolean> CLUSTER_LOGSDB_ENABLED = Setting.boolSetting(
+        "cluster.logsdb.enabled",
+        false,
+        Setting.Property.Dynamic,
+        Setting.Property.NodeScope
+    );
 
     private final LogsdbIndexModeSettingsProvider logsdbIndexModeSettingsProvider;
 
@@ -61,4 +72,12 @@ public Collection<IndexSettingProvider> getAdditionalIndexSettingProviders(Index
     public List<Setting<?>> getSettings() {
         return List.of(FALLBACK_SETTING, CLUSTER_LOGSDB_ENABLED);
     }
+
+    @Override
+    public List<ActionPlugin.ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+        List<ActionPlugin.ActionHandler<? extends ActionRequest, ? extends ActionResponse>> actions = new ArrayList<>();
+        actions.add(new ActionPlugin.ActionHandler<>(XPackUsageFeatureAction.LOGSDB, LogsDBUsageTransportAction.class));
+        actions.add(new ActionPlugin.ActionHandler<>(XPackInfoFeatureAction.LOGSDB, LogsDBInfoTransportAction.class));
+        return actions;
+    }
 }
diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+package org.elasticsearch.xpack.logsdb;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.index.IndexMode;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
+import org.elasticsearch.injection.guice.Inject;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction;
+import org.elasticsearch.xpack.core.application.LogsDBFeatureSetUsage;
+
+import static org.elasticsearch.index.mapper.SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING;
+
+public class LogsDBUsageTransportAction extends XPackUsageFeatureTransportAction {
+    private final ClusterService clusterService;
+
+    @Inject
+    public LogsDBUsageTransportAction(
+        TransportService transportService,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        ActionFilters actionFilters,
+        IndexNameExpressionResolver indexNameExpressionResolver
+    ) {
+        super(
+            XPackUsageFeatureAction.LOGSDB.name(),
+            transportService,
+            clusterService,
+            threadPool,
+            actionFilters,
+            indexNameExpressionResolver
+        );
+        this.clusterService = clusterService;
+    }
+
+    @Override
+    protected void masterOperation(
+        Task task,
+        XPackUsageRequest request,
+        ClusterState state,
+        ActionListener<XPackUsageFeatureResponse> listener
+    ) {
+        int numIndices = 0;
+        int numIndicesWithSyntheticSources = 0;
+        for (IndexMetadata indexMetadata : state.metadata()) {
+            if (indexMetadata.getIndexMode() == IndexMode.LOGSDB) {
+                numIndices++;
+                if (INDEX_MAPPER_SOURCE_MODE_SETTING.get(indexMetadata.getSettings()) == SourceFieldMapper.Mode.SYNTHETIC) {
+                    numIndicesWithSyntheticSources++;
+                }
+            }
+        }
+        final boolean enabled = LogsDBPlugin.CLUSTER_LOGSDB_ENABLED.get(clusterService.getSettings());
+        listener.onResponse(
+            new XPackUsageFeatureResponse(new LogsDBFeatureSetUsage(true, enabled, numIndices, numIndicesWithSyntheticSources))
+        );
+    }
+}
diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java
@@ -10,7 +10,6 @@
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.regex.Regex;
-import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettingProvider;
@@ -20,13 +19,9 @@
 import java.util.List;
 import java.util.Locale;
 
+import static org.elasticsearch.xpack.logsdb.LogsDBPlugin.CLUSTER_LOGSDB_ENABLED;
+
 final class LogsdbIndexModeSettingsProvider implements IndexSettingProvider {
-    static final Setting<Boolean> CLUSTER_LOGSDB_ENABLED = Setting.boolSetting(
-        "cluster.logsdb.enabled",
-        false,
-        Setting.Property.Dynamic,
-        Setting.Property.NodeScope
-    );
     private static final String LOGS_PATTERN = "logs-*-*";
     private volatile boolean isLogsdbEnabled;
 
diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_usage.yml

Original file line number	Diff line number	Diff line change
`@@ -518,6 +518,12 @@ GET /_xpack/usage`
`518`	`518`	`"universal_profiling" : {`
`519`	`519`	`"available" : true,`
`520`	`520`	`"enabled" : true`
	`521`	`+ },`
	`522`	`+ "logsdb": {`
	`523`	`+ "available": true,`
	`524`	`+ "enabled": false,`
	`525`	`+ "indices_count": 0,`
	`526`	`+ "indices_with_synthetic_source": 0`
`521`	`527`	`}`
`522`	`528`	`}`
`523`	`529`	`------------------------------------------------------------`