diff --git a/docs/changelog/134214.yaml b/docs/changelog/134214.yaml new file mode 100644 index 0000000000000..4a66911ae2ea7 --- /dev/null +++ b/docs/changelog/134214.yaml @@ -0,0 +1,6 @@ +pr: 134214 +summary: "[Downsampling++] Add time series telemetry in xpack usage" +area: Downsampling +type: enhancement +issues: + - 133953 diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 9195e23f92ee4..bbabb17549e46 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -344,6 +344,7 @@ static TransportVersion def(int id) { public static final TransportVersion INFERENCE_API_DISABLE_EIS_RATE_LIMITING = def(9_152_0_00); public static final TransportVersion GEMINI_THINKING_BUDGET_ADDED = def(9_153_0_00); public static final TransportVersion VISIT_PERCENTAGE = def(9_154_0_00); + public static final TransportVersion TIME_SERIES_TELEMETRY = def(9_155_0_00); /* * STOP! READ THIS FIRST! No, really, diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportActionIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportActionIT.java new file mode 100644 index 0000000000000..466740398233b --- /dev/null +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportActionIT.java @@ -0,0 +1,533 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.action; + +import org.elasticsearch.action.downsample.DownsampleConfig; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamAlias; +import org.elasticsearch.cluster.metadata.DataStreamLifecycle; +import org.elasticsearch.cluster.metadata.DataStreamOptions; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.datastreams.DataStreamsPlugin; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.protocol.xpack.XPackUsageRequest; +import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xcontent.ToXContent; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.XPackClientPlugin; +import org.elasticsearch.xpack.core.ilm.DownsampleAction; +import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicyMetadata; +import org.elasticsearch.xpack.core.ilm.OperationMode; +import org.elasticsearch.xpack.core.ilm.Phase; +import org.elasticsearch.xpack.core.ilm.ReadOnlyAction; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_DOWNSAMPLE_INTERVAL_KEY; +import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID; +import static org.elasticsearch.xpack.core.action.XPackUsageFeatureAction.TIME_SERIES_DATA_STREAMS; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.nullValue; + +public class TimeSeriesUsageTransportActionIT extends ESIntegTestCase { + private static final String DOWNSAMPLING_IN_HOT_POLICY = "hot-downsampling-policy"; + private static final String DOWNSAMPLING_IN_WARM_COLD_POLICY = "warm-cold-downsampling-policy"; + private static final String NO_DOWNSAMPLING_POLICY = "no-downsampling-policy"; + private static final DateFormatter FORMATTER = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER; + private static final double LIKELIHOOD = 0.8; + + /* + * The TimeSeriesUsageTransportAction is not exposed in the xpack core plugin, so we have a special test plugin to do this. + * We also need to include the DataStreamsPlugin so the data streams will be properly removed by the tests' teardown. + */ + @Override + protected Collection> nodePlugins() { + return List.of(DataStreamsPlugin.class, TestTimeSeriesUsagePlugin.class); + } + + public void testActionNoTimeSeriesDataStreams() throws Exception { + // If the templates are not used, they shouldn't influence the telemetry + if (randomBoolean()) { + updateClusterState(clusterState -> { + Metadata.Builder metadataBuilder = Metadata.builder(clusterState.metadata()); + addIlmPolicies(metadataBuilder); + ClusterState.Builder clusterStateBuilder = new ClusterState.Builder(clusterState); + clusterStateBuilder.metadata(metadataBuilder); + return clusterStateBuilder.build(); + }); + } + Map map = getTimeSeriesUsage(); + assertThat(map.get("available"), equalTo(true)); + assertThat(map.get("enabled"), equalTo(true)); + assertThat(map.get("data_stream_count"), equalTo(0)); + assertThat(map.get("index_count"), nullValue()); + assertThat(map.get("downsampling"), nullValue()); + } + + @SuppressWarnings("unchecked") + public void testAction() throws Exception { + + // Expected counters + // Time series + var timeSeriesDataStreamCount = new AtomicInteger(0); + var timeSeriesIndexCount = new AtomicInteger(0); + + // Downsampling + var downsampled5mIndexCount = new AtomicInteger(0); + var downsampled1hIndexCount = new AtomicInteger(0); + var downsampled1dIndexCount = new AtomicInteger(0); + + // ... with DLM + var dlmDownsampledDataStreamCount = new AtomicInteger(0); + var dlmDownsampledIndexCount = new AtomicInteger(0); + var dlmRoundsCount = new AtomicInteger(0); + var dlmRoundsSum = new AtomicInteger(0); + var dlmRoundsMin = new AtomicInteger(Integer.MAX_VALUE); + var dlmRoundsMax = new AtomicInteger(Integer.MIN_VALUE); + + // ... with ILM + var ilmDownsampledDataStreamCount = new AtomicInteger(0); + var ilmDownsampledIndexCount = new AtomicInteger(0); + var ilmRoundsCount = new AtomicInteger(0); + var ilmRoundsSum = new AtomicInteger(0); + var ilmRoundsMin = new AtomicInteger(Integer.MAX_VALUE); + var ilmRoundsMax = new AtomicInteger(Integer.MIN_VALUE); + Set usedPolicies = new HashSet<>(); + + /* + * We now add a number of simulated data streams to the cluster state. We mix different combinations of: + * - time series and standard data streams & backing indices + * - DLM with or without downsampling + * - ILM with or without downsampling + */ + updateClusterState(clusterState -> { + Metadata.Builder metadataBuilder = Metadata.builder(clusterState.metadata()); + addIlmPolicies(metadataBuilder); + + Map dataStreamMap = new HashMap<>(); + for (int dataStreamCount = 0; dataStreamCount < randomIntBetween(10, 100); dataStreamCount++) { + String dataStreamName = randomAlphaOfLength(50); + boolean isTimeSeriesDataStream = randomBoolean(); + // this flag refers to configuration, a non tsds is also possible to have downsampling + // configuration, but it is skipped. + var downsamplingConfiguredBy = randomFrom(DownsampledBy.values()); + boolean isDownsampled = downsamplingConfiguredBy != DownsampledBy.NONE && isTimeSeriesDataStream; + // An index/data stream can have both ILM & DLM configured; by default, ILM "wins" + boolean hasLifecycle = likely() || (isDownsampled && downsamplingConfiguredBy == DownsampledBy.DLM); + boolean hasIlm = downsamplingConfiguredBy == DownsampledBy.ILM; + + // Replicated data stream should be counted because ILM works independently. + // DLM is not supported yet with CCR. + boolean isReplicated = hasIlm && randomBoolean(); + DataStreamLifecycle lifecycle = maybeCreateLifecycle(isDownsampled, hasLifecycle); + + if (isTimeSeriesDataStream) { + timeSeriesDataStreamCount.incrementAndGet(); + if (downsamplingConfiguredBy == DownsampledBy.DLM) { + dlmDownsampledDataStreamCount.incrementAndGet(); + updateRounds(lifecycle.downsampling().size(), dlmRoundsCount, dlmRoundsSum, dlmRoundsMin, dlmRoundsMax); + } else if (downsamplingConfiguredBy == DownsampledBy.ILM) { + ilmDownsampledDataStreamCount.incrementAndGet(); + } + } + + int backingIndexCount = randomIntBetween(1, 10); + List backingIndices = new ArrayList<>(backingIndexCount); + Instant startTime = Instant.now().minus(randomInt(10), ChronoUnit.DAYS); + for (int i = 0; i < backingIndexCount; i++) { + boolean isWriteIndex = i == backingIndexCount - 1; + Instant endTime = startTime.plus(randomIntBetween(1, 100), ChronoUnit.HOURS); + + // Prepare settings + Settings.Builder settingsBuilder = settings(IndexVersion.current()).put("index.hidden", true) + .put(SETTING_INDEX_UUID, randomUUID()); + + // We do not override DLM if this is the write index + boolean ovewrittenDlm = isWriteIndex == false && rarely() && downsamplingConfiguredBy == DownsampledBy.DLM; + String policy = randomIlmPolicy(downsamplingConfiguredBy, ovewrittenDlm); + if (policy != null) { + settingsBuilder.put("index.lifecycle.name", policy); + } + + // We capture that usually time series data streams have time series indices + // and non-time series data streams have non-time-series indices, but it can + // happen the other way around too + if (isTimeSeriesDataStream && (isWriteIndex || likely()) || isTimeSeriesDataStream == false && rarely()) { + settingsBuilder.put("index.mode", "time_series") + .put("index.time_series.start_time", FORMATTER.format(startTime)) + .put("index.time_series.end_time", FORMATTER.format(endTime)) + .put("index.routing_path", "uid"); + + // The write index cannot be downsampled + // only time series indices can be downsampled + if (isWriteIndex == false && isTimeSeriesDataStream) { + switch (randomIntBetween(0, 3)) { + case 0 -> { + settingsBuilder.put(INDEX_DOWNSAMPLE_INTERVAL_KEY, "5m"); + downsampled5mIndexCount.incrementAndGet(); + } + case 1 -> { + settingsBuilder.put(INDEX_DOWNSAMPLE_INTERVAL_KEY, "1h"); + downsampled1hIndexCount.incrementAndGet(); + } + case 2 -> { + settingsBuilder.put(INDEX_DOWNSAMPLE_INTERVAL_KEY, "1d"); + downsampled1dIndexCount.incrementAndGet(); + } + default -> { + } + } + } + // If the data stream is not time series we do not count the backing indices. + if (isTimeSeriesDataStream) { + timeSeriesIndexCount.incrementAndGet(); + if (downsamplingConfiguredBy == DownsampledBy.ILM || ovewrittenDlm) { + ilmDownsampledIndexCount.incrementAndGet(); + usedPolicies.add(policy); + if (isWriteIndex) { + updateRounds( + DOWNSAMPLING_IN_HOT_POLICY.equals(policy) ? 1 : 2, + ilmRoundsCount, + ilmRoundsSum, + ilmRoundsMin, + ilmRoundsMax + ); + } + } else if (downsamplingConfiguredBy == DownsampledBy.DLM) { + dlmDownsampledIndexCount.incrementAndGet(); + } + } + } + IndexMetadata indexMetadata = IndexMetadata.builder(DataStream.getDefaultBackingIndexName(dataStreamName, i + 1)) + .settings(settingsBuilder) + .numberOfShards(1) + .numberOfReplicas(0) + .build(); + startTime = endTime; + backingIndices.add(indexMetadata.getIndex()); + metadataBuilder.put(indexMetadata, true); + } + DataStream dataStream = new DataStream( + dataStreamName, + backingIndices, + randomLongBetween(0, 1000), + Map.of(), + randomBoolean(), + isReplicated, + false, + randomBoolean(), + isTimeSeriesDataStream ? IndexMode.TIME_SERIES : IndexMode.STANDARD, + lifecycle, + DataStreamOptions.EMPTY, + List.of(), + isReplicated == false && randomBoolean(), + null + ); + dataStreamMap.put(dataStream.getName(), dataStream); + } + Map dataStreamAliasesMap = Map.of(); + metadataBuilder.dataStreams(dataStreamMap, dataStreamAliasesMap); + ClusterState.Builder clusterStateBuilder = new ClusterState.Builder(clusterState); + clusterStateBuilder.metadata(metadataBuilder); + return clusterStateBuilder.build(); + }); + + Map map = getTimeSeriesUsage(); + assertThat(map.get("available"), equalTo(true)); + assertThat(map.get("enabled"), equalTo(true)); + assertThat(map.get("data_stream_count"), equalTo(timeSeriesDataStreamCount.get())); + if (timeSeriesDataStreamCount.get() == 0) { + assertThat(map.get("index_count"), nullValue()); + assertThat(map.get("downsampling"), nullValue()); + } else { + assertThat(map.get("index_count"), equalTo(timeSeriesIndexCount.get())); + assertThat(map.containsKey("downsampling"), equalTo(true)); + Map downsamplingMap = (Map) map.get("downsampling"); + + // Downsampled indices + assertDownsampledIndices( + downsamplingMap, + downsampled5mIndexCount.get(), + downsampled1hIndexCount.get(), + downsampled1dIndexCount.get() + ); + + // DLM + assertThat(downsamplingMap.containsKey("dlm"), equalTo(true)); + assertDownsamplingStats( + (Map) downsamplingMap.get("dlm"), + dlmDownsampledDataStreamCount.get(), + dlmDownsampledIndexCount.get(), + dlmRoundsCount.get(), + dlmRoundsSum.get(), + dlmRoundsMin.get(), + dlmRoundsMax.get() + ); + + // ILM + assertThat(downsamplingMap.containsKey("ilm"), equalTo(true)); + Map ilmStats = (Map) downsamplingMap.get("ilm"); + assertDownsamplingStats( + ilmStats, + ilmDownsampledDataStreamCount.get(), + ilmDownsampledIndexCount.get(), + ilmRoundsCount.get(), + ilmRoundsSum.get(), + ilmRoundsMin.get(), + ilmRoundsMax.get() + ); + Map phasesStats = (Map) ilmStats.get("phases_in_use"); + if (usedPolicies.contains(DOWNSAMPLING_IN_HOT_POLICY)) { + assertThat(phasesStats.get("hot"), equalTo(1)); + } else { + assertThat(phasesStats.get("hot"), nullValue()); + } + if (usedPolicies.contains(DOWNSAMPLING_IN_WARM_COLD_POLICY)) { + assertThat(phasesStats.get("warm"), equalTo(1)); + assertThat(phasesStats.get("cold"), equalTo(1)); + } else { + assertThat(phasesStats.get("warm"), nullValue()); + assertThat(phasesStats.get("cold"), nullValue()); + } + + } + + } + + @SuppressWarnings("unchecked") + private static void assertDownsampledIndices( + Map downsamplingMap, + int downsampled5mIndexCount, + int downsampled1hIndexCount, + int downsampled1dIndexCount + ) { + if (downsampled5mIndexCount > 0 || downsampled1hIndexCount > 0 || downsampled1dIndexCount > 0) { + assertThat(downsamplingMap.containsKey("index_count_per_interval"), equalTo(true)); + Map downsampledIndicesMap = (Map) downsamplingMap.get("index_count_per_interval"); + if (downsampled5mIndexCount > 0) { + assertThat(downsampledIndicesMap.get("5m"), equalTo(downsampled5mIndexCount)); + } else { + assertThat(downsampledIndicesMap.get("5m"), nullValue()); + } + if (downsampled1hIndexCount > 0) { + assertThat(downsampledIndicesMap.get("1h"), equalTo(downsampled1hIndexCount)); + } else { + assertThat(downsampledIndicesMap.get("1h"), nullValue()); + } + if (downsampled1dIndexCount > 0) { + assertThat(downsampledIndicesMap.get("1d"), equalTo(downsampled1dIndexCount)); + } else { + assertThat(downsampledIndicesMap.get("1d"), nullValue()); + } + } + } + + @SuppressWarnings("unchecked") + private void assertDownsamplingStats( + Map stats, + Integer downsampledDataStreamCount, + Integer downsampledIndexCount, + Integer roundsCount, + Integer roundsSum, + Integer roundsMin, + Integer roundsMax + ) { + assertThat(stats.get("downsampled_data_stream_count"), equalTo(downsampledDataStreamCount)); + if (downsampledDataStreamCount == 0) { + assertThat(stats.get("downsampled_index_count"), nullValue()); + assertThat(stats.get("rounds_per_data_stream"), nullValue()); + } else { + assertThat(stats.get("downsampled_index_count"), equalTo(downsampledIndexCount)); + assertThat(stats.containsKey("rounds_per_data_stream"), equalTo(true)); + Map roundsMap = (Map) stats.get("rounds_per_data_stream"); + assertThat(roundsMap.get("average"), equalTo((double) roundsSum / roundsCount)); + assertThat(roundsMap.get("min"), equalTo(roundsMin)); + assertThat(roundsMap.get("max"), equalTo(roundsMax)); + } + } + + private DataStreamLifecycle maybeCreateLifecycle(boolean isDownsampled, boolean hasDlm) { + if (hasDlm == false) { + return null; + } + var builder = DataStreamLifecycle.dataLifecycleBuilder(); + if (isDownsampled) { + builder.downsampling(randomDownsamplingRounds()); + } + return builder.build(); + } + + private void updateRounds( + int size, + AtomicInteger roundsCount, + AtomicInteger roundsSum, + AtomicInteger roundsMin, + AtomicInteger roundsMax + ) { + roundsCount.incrementAndGet(); + roundsSum.addAndGet(size); + roundsMin.updateAndGet(v -> Math.min(v, size)); + roundsMax.updateAndGet(v -> Math.max(v, size)); + } + + private Map getTimeSeriesUsage() throws IOException { + XPackUsageFeatureResponse response = safeGet(client().execute(TIME_SERIES_DATA_STREAMS, new XPackUsageRequest(SAFE_AWAIT_TIMEOUT))); + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder = response.getUsage().toXContent(builder, ToXContent.EMPTY_PARAMS); + Tuple> tuple = XContentHelper.convertToMap( + BytesReference.bytes(builder), + true, + XContentType.JSON + ); + return tuple.v2(); + } + + /* + * Updates the cluster state in the internal cluster using the provided function + */ + protected static void updateClusterState(final Function updater) throws Exception { + final PlainActionFuture future = new PlainActionFuture<>(); + + final ClusterService clusterService = internalCluster().getCurrentMasterNodeInstance(ClusterService.class); + clusterService.submitUnbatchedStateUpdateTask("time-series-usage-test", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) { + return updater.apply(currentState); + } + + @Override + public void onFailure(Exception e) { + future.onFailure(e); + } + + @Override + public void clusterStateProcessed(ClusterState oldState, ClusterState newState) { + future.onResponse(null); + } + }); + safeGet(future); + } + + private List randomDownsamplingRounds() { + List rounds = new ArrayList<>(); + int minutes = 5; + int days = 1; + for (int i = 0; i < randomIntBetween(1, 10); i++) { + rounds.add( + new DataStreamLifecycle.DownsamplingRound( + TimeValue.timeValueDays(days), + new DownsampleConfig(new DateHistogramInterval(minutes + "m")) + ) + ); + minutes *= randomIntBetween(2, 5); + days += randomIntBetween(1, 5); + } + return rounds; + } + + private void addIlmPolicies(Metadata.Builder metadataBuilder) { + List policies = List.of( + new LifecyclePolicy( + DOWNSAMPLING_IN_HOT_POLICY, + Map.of( + "hot", + new Phase("hot", TimeValue.ZERO, Map.of("downsample", new DownsampleAction(DateHistogramInterval.MINUTE, null))) + ) + ), + new LifecyclePolicy( + DOWNSAMPLING_IN_WARM_COLD_POLICY, + Map.of( + "warm", + new Phase("warm", TimeValue.ZERO, Map.of("downsample", new DownsampleAction(DateHistogramInterval.HOUR, null))), + "cold", + new Phase( + "cold", + TimeValue.timeValueDays(3), + Map.of("downsample", new DownsampleAction(DateHistogramInterval.DAY, null)) + ) + ) + ), + new LifecyclePolicy( + NO_DOWNSAMPLING_POLICY, + Map.of("hot", new Phase("hot", TimeValue.ZERO, Map.of("read-only", new ReadOnlyAction()))) + ) + ); + Map policyMetadata = policies.stream() + .collect( + Collectors.toMap( + LifecyclePolicy::getName, + lifecyclePolicy -> new LifecyclePolicyMetadata(lifecyclePolicy, Map.of(), 1, Instant.now().toEpochMilli()) + ) + ); + IndexLifecycleMetadata newMetadata = new IndexLifecycleMetadata(policyMetadata, OperationMode.RUNNING); + metadataBuilder.putCustom(IndexLifecycleMetadata.TYPE, newMetadata); + } + + private static String randomIlmPolicy(DownsampledBy downsampledBy, boolean ovewrittenDlm) { + if (downsampledBy == DownsampledBy.ILM || (downsampledBy == DownsampledBy.DLM && ovewrittenDlm)) { + return randomFrom(DOWNSAMPLING_IN_HOT_POLICY, DOWNSAMPLING_IN_WARM_COLD_POLICY); + } + if (downsampledBy == DownsampledBy.NONE && likely()) { + return NO_DOWNSAMPLING_POLICY; + } + return null; + } + + private static boolean likely() { + return randomDoubleBetween(0.0, 1.0, true) < LIKELIHOOD; + } + + private enum DownsampledBy { + DLM, + ILM, + NONE + } + + /* + * This plugin exposes the TimeSeriesUsageTransportAction. + */ + public static final class TestTimeSeriesUsagePlugin extends XPackClientPlugin { + + @Override + public List getActions() { + return List.of(new ActionHandler(TIME_SERIES_DATA_STREAMS, TimeSeriesUsageTransportAction.class)); + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java index 972d949a9010f..ca55031a1f5c9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java @@ -74,6 +74,8 @@ public final class XPackField { public static final String DATA_STREAMS = "data_streams"; /** Name constant for the data stream lifecycle feature. */ public static final String DATA_STREAM_LIFECYCLE = "data_lifecycle"; + /** Name constant for the time series data streams feature. */ + public static final String TIME_SERIES_DATA_STREAMS = "time_series"; /** Name constant for the data tiers feature. */ public static final String DATA_TIERS = "data_tiers"; /** Name constant for the aggregate_metric plugin. */ diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java index bb0a21a6afd2a..8ab5a3139f49b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java @@ -91,6 +91,7 @@ import org.elasticsearch.xpack.core.action.DataStreamInfoTransportAction; import org.elasticsearch.xpack.core.action.DataStreamLifecycleUsageTransportAction; import org.elasticsearch.xpack.core.action.DataStreamUsageTransportAction; +import org.elasticsearch.xpack.core.action.TimeSeriesUsageTransportAction; import org.elasticsearch.xpack.core.action.TransportXPackInfoAction; import org.elasticsearch.xpack.core.action.TransportXPackUsageAction; import org.elasticsearch.xpack.core.action.XPackInfoAction; @@ -374,6 +375,7 @@ public List getActions() { actions.add(new ActionHandler(XPackUsageFeatureAction.HEALTH, HealthApiUsageTransportAction.class)); actions.add(new ActionHandler(XPackUsageFeatureAction.REMOTE_CLUSTERS, RemoteClusterUsageTransportAction.class)); actions.add(new ActionHandler(NodesDataTiersUsageTransportAction.TYPE, NodesDataTiersUsageTransportAction.class)); + actions.add(new ActionHandler(XPackUsageFeatureAction.TIME_SERIES_DATA_STREAMS, TimeSeriesUsageTransportAction.class)); return actions; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportAction.java new file mode 100644 index 0000000000000..ec812f30bc6a7 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TimeSeriesUsageTransportAction.java @@ -0,0 +1,193 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamLifecycle; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.ProjectMetadata; +import org.elasticsearch.cluster.project.ProjectResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.protocol.xpack.XPackUsageRequest; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.datastreams.TimeSeriesFeatureSetUsage; +import org.elasticsearch.xpack.core.ilm.DownsampleAction; +import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicyMetadata; +import org.elasticsearch.xpack.core.ilm.Phase; + +import java.util.HashMap; +import java.util.LongSummaryStatistics; +import java.util.Map; +import java.util.Objects; + +/** + * Exposes the time series telemetry via the xpack usage API. We track the following only for time series data streams: + * - time series data stream count + * - time series backing indices of these time series data streams + * - the feature that downsamples the time series data streams, we use the write index to avoid resolving templates, + * this might cause a small delay in the counters (backing indices, downsampling rounds). + * - For ILM specifically, we count the phases that have configured downsampling in the policies used in the time series data streams. + * - When elasticsearch is running in DLM only mode, we skip all the ILM metrics. + */ +public class TimeSeriesUsageTransportAction extends XPackUsageFeatureTransportAction { + + private final ProjectResolver projectResolver; + private final boolean ilmAvailable; + + @Inject + public TimeSeriesUsageTransportAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + ProjectResolver projectResolver + ) { + super(XPackUsageFeatureAction.TIME_SERIES_DATA_STREAMS.name(), transportService, clusterService, threadPool, actionFilters); + this.projectResolver = projectResolver; + this.ilmAvailable = DataStreamLifecycle.isDataStreamsLifecycleOnlyMode(clusterService.getSettings()) == false; + } + + @Override + protected void localClusterStateOperation( + Task task, + XPackUsageRequest request, + ClusterState state, + ActionListener listener + ) { + ProjectMetadata projectMetadata = projectResolver.getProjectMetadata(state); + IndexLifecycleMetadata ilmMetadata = projectMetadata.custom(IndexLifecycleMetadata.TYPE, IndexLifecycleMetadata.EMPTY); + final Map dataStreams = projectMetadata.dataStreams(); + + long tsDataStreamCount = 0; + long tsIndexCount = 0; + IlmDownsamplingStatsTracker ilmStats = ilmAvailable ? new IlmDownsamplingStatsTracker() : null; + DownsamplingStatsTracker dlmStats = new DownsamplingStatsTracker(); + Map indicesByInterval = new HashMap<>(); + + for (DataStream ds : dataStreams.values()) { + // We choose to not count time series backing indices that do not belong to a time series data stream. + if (ds.getIndexMode() != IndexMode.TIME_SERIES) { + continue; + } + tsDataStreamCount++; + Integer dlmRounds = ds.getDataLifecycle() == null || ds.getDataLifecycle().downsampling() == null + ? null + : ds.getDataLifecycle().downsampling().size(); + + for (Index backingIndex : ds.getIndices()) { + IndexMetadata indexMetadata = projectMetadata.index(backingIndex); + if (indexMetadata.getIndexMode() != IndexMode.TIME_SERIES) { + continue; + } + tsIndexCount++; + if (ds.isIndexManagedByDataStreamLifecycle(indexMetadata.getIndex(), ignored -> indexMetadata) && dlmRounds != null) { + dlmStats.trackIndex(ds, indexMetadata); + dlmStats.trackRounds(dlmRounds, ds, indexMetadata); + } else if (ilmAvailable && projectMetadata.isIndexManagedByILM(indexMetadata)) { + LifecyclePolicyMetadata policyMetadata = ilmMetadata.getPolicyMetadatas().get(indexMetadata.getLifecyclePolicyName()); + if (policyMetadata == null) { + continue; + } + int rounds = 0; + for (Phase phase : policyMetadata.getPolicy().getPhases().values()) { + if (phase.getActions().containsKey(DownsampleAction.NAME)) { + rounds++; + } + } + if (rounds > 0) { + ilmStats.trackPolicy(policyMetadata.getPolicy()); + ilmStats.trackIndex(ds, indexMetadata); + ilmStats.trackRounds(rounds, ds, indexMetadata); + } + } + String interval = indexMetadata.getSettings().get(IndexMetadata.INDEX_DOWNSAMPLE_INTERVAL.getKey()); + if (interval != null) { + Long count = indicesByInterval.computeIfAbsent(interval, ignored -> 0L); + indicesByInterval.put(interval, count + 1); + } + } + } + + final TimeSeriesFeatureSetUsage usage = ilmAvailable + ? new TimeSeriesFeatureSetUsage( + tsDataStreamCount, + tsIndexCount, + ilmStats.getDownsamplingStats(), + ilmStats.getIlmPolicyStats(), + dlmStats.getDownsamplingStats(), + indicesByInterval + ) + : new TimeSeriesFeatureSetUsage(tsDataStreamCount, tsIndexCount, dlmStats.getDownsamplingStats(), indicesByInterval); + listener.onResponse(new XPackUsageFeatureResponse(usage)); + } + + private static class DownsamplingStatsTracker { + private long downsampledDataStreams = 0; + private long downsampledIndices = 0; + private final LongSummaryStatistics rounds = new LongSummaryStatistics(); + + void trackIndex(DataStream ds, IndexMetadata indexMetadata) { + if (Objects.equals(indexMetadata.getIndex(), ds.getWriteIndex())) { + downsampledDataStreams++; + } + downsampledIndices++; + } + + void trackRounds(int rounds, DataStream ds, IndexMetadata indexMetadata) { + // We want to track rounds per data stream, so we use the write index to determine the + // rounds applicable for this data stream + if (Objects.equals(indexMetadata.getIndex(), ds.getWriteIndex())) { + this.rounds.accept(rounds); + } + } + + TimeSeriesFeatureSetUsage.DownsamplingFeatureStats getDownsamplingStats() { + return new TimeSeriesFeatureSetUsage.DownsamplingFeatureStats( + downsampledDataStreams, + downsampledIndices, + rounds.getMin(), + rounds.getAverage(), + rounds.getMax() + ); + } + } + + static class IlmDownsamplingStatsTracker extends DownsamplingStatsTracker { + private final Map> policies = new HashMap<>(); + + void trackPolicy(LifecyclePolicy ilmPolicy) { + policies.putIfAbsent(ilmPolicy.getName(), ilmPolicy.getPhases()); + } + + Map getIlmPolicyStats() { + if (policies.isEmpty()) { + return Map.of(); + } + Map downsamplingPhases = new HashMap<>(); + for (String ilmPolicy : policies.keySet()) { + for (Phase phase : policies.get(ilmPolicy).values()) { + if (phase.getActions().containsKey(DownsampleAction.NAME)) { + Long current = downsamplingPhases.computeIfAbsent(phase.getName(), ignored -> 0L); + downsamplingPhases.put(phase.getName(), current + 1); + } + } + } + return downsamplingPhases; + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java index 455d0d1e2377e..8d0d3ebadfb6a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java @@ -53,6 +53,9 @@ private XPackUsageFeatureAction() {/* no instances */} public static final ActionType DATA_STREAM_LIFECYCLE = xpackUsageFeatureAction( XPackField.DATA_STREAM_LIFECYCLE ); + public static final ActionType TIME_SERIES_DATA_STREAMS = xpackUsageFeatureAction( + XPackField.TIME_SERIES_DATA_STREAMS + ); public static final ActionType DATA_TIERS = xpackUsageFeatureAction(XPackField.DATA_TIERS); public static final ActionType AGGREGATE_METRIC = xpackUsageFeatureAction(XPackField.AGGREGATE_METRIC); public static final ActionType ARCHIVE = xpackUsageFeatureAction(XPackField.ARCHIVE); @@ -91,7 +94,8 @@ private XPackUsageFeatureAction() {/* no instances */} REMOTE_CLUSTERS, ENTERPRISE_SEARCH, UNIVERSAL_PROFILING, - LOGSDB + LOGSDB, + TIME_SERIES_DATA_STREAMS ); public static ActionType xpackUsageFeatureAction(String suffix) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datastreams/TimeSeriesFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datastreams/TimeSeriesFeatureSetUsage.java new file mode 100644 index 0000000000000..6e354f0896151 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datastreams/TimeSeriesFeatureSetUsage.java @@ -0,0 +1,284 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.datastreams; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xcontent.ToXContentFragment; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.XPackFeatureUsage; +import org.elasticsearch.xpack.core.XPackField; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + +/** + * Telemetry for time series data, only time series data streams (TSDS) are tracked. For each TSDS we track: + * - their time series backing indices + * - their downsampled backing indices + * - the downsampled data streams, backing indices and downsampling rounds split by feature (ILM or DLM) + * - for ILM specifically, we count also the phase in which the downsampling round was configured only for + * policies used by said data streams + * { + * "time_series": { + * "enabled": true, + * "available": true, + * "data_streams_count": 10, + * "indices_count": 100, + * "downsampling": { + * "index_count_per_interval": { + * "5m": 5, + * "10m": 10, + * "1h": 10000 + * }, + * "ilm": { + * "downsampled_data_stream_count": 8, + * "downsampled_index_count": 50, + * "rounds_per_data_stream": { + * "min": 1, + * "max": 3, + * "average": 2 + * }, + * "phases_in_use": { + * "hot": 10, + * "warm": 5, + * "cold": 10 + * } + * }, + * "dlm": { + * "downsampled_data_stream_count": 8, + * "downsampled_index_count": 50, + * "rounds_per_data_stream": { + * "min": 1, + * "max": 3, + * "average": 2 + * } + * } + * } + * } + * } + */ +public class TimeSeriesFeatureSetUsage extends XPackFeatureUsage { + private final long timeSeriesDataStreamCount; + private final long timeSeriesIndexCount; + private final DownsamplingUsage downsamplingUsage; + + public TimeSeriesFeatureSetUsage(StreamInput input) throws IOException { + super(input); + this.timeSeriesDataStreamCount = input.readVLong(); + if (timeSeriesDataStreamCount == 0) { + timeSeriesIndexCount = 0; + downsamplingUsage = null; + } else { + this.timeSeriesIndexCount = input.readVLong(); + this.downsamplingUsage = input.readOptionalWriteable(DownsamplingUsage::read); + } + } + + /** + * Helper constructor that only requires DLM stats. This can be used when elasticsearch is running in + * data-stream-lifecycle-only mode. In this mode ILM is not supported, which entails there will be no stats either. + */ + public TimeSeriesFeatureSetUsage( + long timeSeriesDataStreamCount, + long timeSeriesIndexCount, + DownsamplingFeatureStats dlmDownsamplingStats, + Map indexCountPerInterval + ) { + this(timeSeriesDataStreamCount, timeSeriesIndexCount, null, null, dlmDownsamplingStats, indexCountPerInterval); + } + + public TimeSeriesFeatureSetUsage( + long timeSeriesDataStreamCount, + long timeSeriesIndexCount, + DownsamplingFeatureStats ilmDownsamplingStats, + Map phasesUsedInDownsampling, + DownsamplingFeatureStats dlmDownsamplingStats, + Map indexCountPerInterval + ) { + super(XPackField.TIME_SERIES_DATA_STREAMS, true, true); + this.timeSeriesDataStreamCount = timeSeriesDataStreamCount; + if (timeSeriesDataStreamCount == 0) { + this.timeSeriesIndexCount = 0; + this.downsamplingUsage = null; + } else { + this.timeSeriesIndexCount = timeSeriesIndexCount; + this.downsamplingUsage = new DownsamplingUsage( + ilmDownsamplingStats, + phasesUsedInDownsampling, + dlmDownsamplingStats, + indexCountPerInterval + ); + } + + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeVLong(timeSeriesDataStreamCount); + if (timeSeriesDataStreamCount > 0) { + out.writeVLong(timeSeriesIndexCount); + out.writeOptionalWriteable(downsamplingUsage); + } + + } + + @Override + public TransportVersion getMinimalSupportedVersion() { + return TransportVersions.TIME_SERIES_TELEMETRY; + } + + public long getTimeSeriesDataStreamCount() { + return timeSeriesDataStreamCount; + } + + public long getTimeSeriesIndexCount() { + return timeSeriesIndexCount; + } + + public DownsamplingUsage getDownsamplingUsage() { + return downsamplingUsage; + } + + @Override + protected void innerXContent(XContentBuilder builder, Params params) throws IOException { + super.innerXContent(builder, params); + builder.field("data_stream_count", timeSeriesDataStreamCount); + if (timeSeriesDataStreamCount > 0) { + builder.field("index_count", timeSeriesIndexCount); + } + if (downsamplingUsage != null) { + builder.field("downsampling", downsamplingUsage); + } + } + + @Override + public String toString() { + return Strings.toString(this); + } + + @Override + public int hashCode() { + return Objects.hash(timeSeriesDataStreamCount, timeSeriesIndexCount, downsamplingUsage); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (obj.getClass() != getClass()) { + return false; + } + TimeSeriesFeatureSetUsage other = (TimeSeriesFeatureSetUsage) obj; + return timeSeriesDataStreamCount == other.timeSeriesDataStreamCount + && timeSeriesIndexCount == other.timeSeriesIndexCount + && Objects.equals(downsamplingUsage, other.downsamplingUsage); + } + + public record DownsamplingUsage( + DownsamplingFeatureStats ilmDownsamplingStats, + Map phasesUsedInDownsampling, + DownsamplingFeatureStats dlmDownsamplingStats, + Map indexCountPerInterval + ) implements Writeable, ToXContentObject { + + public static DownsamplingUsage read(StreamInput in) throws IOException { + DownsamplingFeatureStats ilmDownsamplingStats = in.readOptionalWriteable(DownsamplingFeatureStats::read); + Map phasesUsedInDownsampling = ilmDownsamplingStats != null + ? in.readImmutableMap(StreamInput::readString, StreamInput::readVLong) + : null; + DownsamplingFeatureStats dlmDownsamplingStats = DownsamplingFeatureStats.read(in); + Map indexCountPerInterval = in.readImmutableMap(StreamInput::readString, StreamInput::readVLong); + return new DownsamplingUsage(ilmDownsamplingStats, phasesUsedInDownsampling, dlmDownsamplingStats, indexCountPerInterval); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalWriteable(ilmDownsamplingStats); + if (ilmDownsamplingStats != null) { + out.writeMap(phasesUsedInDownsampling, StreamOutput::writeString, StreamOutput::writeVLong); + } + dlmDownsamplingStats.writeTo(out); + out.writeMap(indexCountPerInterval, StreamOutput::writeString, StreamOutput::writeVLong); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (indexCountPerInterval != null && indexCountPerInterval.isEmpty() == false) { + builder.startObject("index_count_per_interval"); + for (Map.Entry entry : indexCountPerInterval.entrySet()) { + builder.field(entry.getKey(), entry.getValue()); + } + builder.endObject(); + } + if (ilmDownsamplingStats != null) { + builder.startObject("ilm"); + ilmDownsamplingStats.toXContent(builder, params); + builder.field("phases_in_use", phasesUsedInDownsampling); + builder.endObject(); + } + if (dlmDownsamplingStats != null) { + builder.startObject("dlm"); + dlmDownsamplingStats.toXContent(builder, params); + builder.endObject(); + } + return builder.endObject(); + } + } + + public record DownsamplingFeatureStats(long dataStreamsCount, long indexCount, long minRounds, double averageRounds, long maxRounds) + implements + Writeable, + ToXContentFragment { + + static final DownsamplingFeatureStats EMPTY = new DownsamplingFeatureStats(0, 0, 0, 0.0, 0); + + public static DownsamplingFeatureStats read(StreamInput in) throws IOException { + long dataStreamsCount = in.readVLong(); + if (dataStreamsCount == 0) { + return EMPTY; + } else { + return new DownsamplingFeatureStats(dataStreamsCount, in.readVLong(), in.readVLong(), in.readDouble(), in.readVLong()); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(this.dataStreamsCount); + if (this.dataStreamsCount != 0) { + out.writeVLong(this.indexCount); + out.writeVLong(this.minRounds); + out.writeDouble(this.averageRounds); + out.writeVLong(this.maxRounds); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("downsampled_data_stream_count", dataStreamsCount); + if (dataStreamsCount > 0) { + builder.field("downsampled_index_count", indexCount); + builder.startObject("rounds_per_data_stream"); + builder.field("min", minRounds); + builder.field("average", averageRounds); + builder.field("max", maxRounds); + builder.endObject(); + } + return builder; + } + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datastreams/TimeSeriesFeatureSetUsageTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datastreams/TimeSeriesFeatureSetUsageTests.java new file mode 100644 index 0000000000000..3f1d3983ddbfa --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datastreams/TimeSeriesFeatureSetUsageTests.java @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.datastreams; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.io.IOException; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class TimeSeriesFeatureSetUsageTests extends AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return TimeSeriesFeatureSetUsage::new; + } + + @Override + protected TimeSeriesFeatureSetUsage createTestInstance() { + int randomisationBranch = randomIntBetween(0, 3); + return switch (randomisationBranch) { + case 0 -> new TimeSeriesFeatureSetUsage(0, 0, null, null); + case 1 -> new TimeSeriesFeatureSetUsage( + randomIntBetween(0, 100), + randomIntBetween(100, 100000), + TimeSeriesFeatureSetUsage.DownsamplingFeatureStats.EMPTY, + Map.of() + ); + case 2 -> new TimeSeriesFeatureSetUsage( + randomIntBetween(0, 100), + randomIntBetween(100, 100000), + randomDownsamplingFeatureStats(), + randomPhaseMap(), + randomDownsamplingFeatureStats(), + randomIntervalMap() + ); + case 3 -> new TimeSeriesFeatureSetUsage( + randomIntBetween(0, 100), + randomIntBetween(100, 100000), + randomDownsamplingFeatureStats(), + randomIntervalMap() + ); + default -> throw new AssertionError("Illegal randomisation branch: " + randomisationBranch); + }; + } + + @Override + protected TimeSeriesFeatureSetUsage mutateInstance(TimeSeriesFeatureSetUsage instance) throws IOException { + var dataStreamCount = instance.getTimeSeriesDataStreamCount(); + if (dataStreamCount == 0) { + return new TimeSeriesFeatureSetUsage( + randomIntBetween(0, 100), + randomIntBetween(100, 100000), + TimeSeriesFeatureSetUsage.DownsamplingFeatureStats.EMPTY, + Map.of() + ); + } + var indexCount = instance.getTimeSeriesIndexCount(); + var ilm = instance.getDownsamplingUsage().ilmDownsamplingStats(); + var ilmPhases = instance.getDownsamplingUsage().phasesUsedInDownsampling(); + var dlm = instance.getDownsamplingUsage().dlmDownsamplingStats(); + var indexPerInterval = instance.getDownsamplingUsage().indexCountPerInterval(); + int randomisationBranch = between(0, 5); + switch (randomisationBranch) { + case 0 -> dataStreamCount += randomIntBetween(1, 100); + case 1 -> indexCount += randomIntBetween(1, 100); + case 2 -> ilm = randomValueOtherThan(ilm, this::randomDownsamplingFeatureStats); + case 3 -> ilmPhases = randomValueOtherThan(ilmPhases, this::randomPhaseMap); + case 4 -> dlm = randomValueOtherThan(dlm, this::randomDownsamplingFeatureStats); + case 5 -> indexPerInterval = randomValueOtherThan(indexPerInterval, this::randomIntervalMap); + default -> throw new AssertionError("Illegal randomisation branch: " + randomisationBranch); + } + return new TimeSeriesFeatureSetUsage(dataStreamCount, indexCount, ilm, ilmPhases, dlm, indexPerInterval); + } + + private TimeSeriesFeatureSetUsage.DownsamplingFeatureStats randomDownsamplingFeatureStats() { + return new TimeSeriesFeatureSetUsage.DownsamplingFeatureStats( + randomIntBetween(1, 100), + randomIntBetween(1, 100), + randomIntBetween(1, 10), + randomDoubleBetween(1.0, 10.0, true), + randomIntBetween(1, 10) + ); + } + + private Map randomPhaseMap() { + return randomNonEmptySubsetOf(Set.of("hot", "warm", "cold")).stream() + .collect(Collectors.toMap(k -> k, ignored -> randomNonNegativeLong())); + } + + private Map randomIntervalMap() { + return randomMap(1, 10, () -> Tuple.tuple(randomIntBetween(1, 400) + randomFrom("m", "h", "d"), randomNonNegativeLong())); + } +} diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index 485b6989aca58..da9a81898de60 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -487,6 +487,7 @@ public class Constants { "cluster:monitor/xpack/usage/universal_profiling", "cluster:monitor/xpack/usage/voting_only", "cluster:monitor/xpack/usage/watcher", + "cluster:monitor/xpack/usage/time_series", "cluster:monitor/xpack/watcher/stats/dist", "cluster:monitor/xpack/watcher/watch/get", "cluster:monitor/xpack/watcher/watch/query",