Add has_custom_cutoff_date to logsdb usage. (#117550) (#117610)

martijnvg · web-flow · commit a42b8431ecfc · 2024-11-27T18:48:42.000+11:00
Indicates whether es.mapping.synthetic_source_fallback_to_stored_source.cutoff_date_restricted_override system property has been configured. A follow up from #116647
diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java
@@ -204,9 +204,11 @@ static TransportVersion def(int id) {
     public static final TransportVersion FAST_REFRESH_RCO_2 = def(8_795_00_0);
     public static final TransportVersion ESQL_ENRICH_RUNTIME_WARNINGS = def(8_796_00_0);
     public static final TransportVersion INGEST_PIPELINE_CONFIGURATION_AS_MAP = def(8_797_00_0);
+    public static final TransportVersion LOGSDB_TELEMETRY_CUSTOM_CUTOFF_DATE_FIX_8_17 = def(8_797_00_1);
     public static final TransportVersion INDEXING_PRESSURE_THROTTLING_STATS = def(8_798_00_0);
     public static final TransportVersion REINDEX_DATA_STREAMS = def(8_799_00_0);
     public static final TransportVersion ESQL_REMOVE_NODE_LEVEL_PLAN = def(8_800_00_0);
+    public static final TransportVersion LOGSDB_TELEMETRY_CUSTOM_CUTOFF_DATE = def(8_801_00_0);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/application/LogsDBFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/application/LogsDBFeatureSetUsage.java
@@ -22,6 +22,7 @@ public final class LogsDBFeatureSetUsage extends XPackFeatureUsage {
     private final int indicesWithSyntheticSource;
     private final long numDocs;
     private final long sizeInBytes;
+    private final boolean hasCustomCutoffDate;
 
     public LogsDBFeatureSetUsage(StreamInput input) throws IOException {
         super(input);
@@ -34,6 +35,13 @@ public LogsDBFeatureSetUsage(StreamInput input) throws IOException {
             numDocs = 0;
             sizeInBytes = 0;
         }
+        var transportVersion = input.getTransportVersion();
+        if (transportVersion.isPatchFrom(TransportVersions.LOGSDB_TELEMETRY_CUSTOM_CUTOFF_DATE_FIX_8_17)
+            || transportVersion.onOrAfter(TransportVersions.LOGSDB_TELEMETRY_CUSTOM_CUTOFF_DATE)) {
+            hasCustomCutoffDate = input.readBoolean();
+        } else {
+            hasCustomCutoffDate = false;
+        }
     }
 
     @Override
@@ -45,6 +53,11 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeVLong(numDocs);
             out.writeVLong(sizeInBytes);
         }
+        var transportVersion = out.getTransportVersion();
+        if (transportVersion.isPatchFrom(TransportVersions.LOGSDB_TELEMETRY_CUSTOM_CUTOFF_DATE_FIX_8_17)
+            || transportVersion.onOrAfter(TransportVersions.LOGSDB_TELEMETRY_CUSTOM_CUTOFF_DATE)) {
+            out.writeBoolean(hasCustomCutoffDate);
+        }
     }
 
     public LogsDBFeatureSetUsage(
@@ -53,13 +66,15 @@ public LogsDBFeatureSetUsage(
         int indicesCount,
         int indicesWithSyntheticSource,
         long numDocs,
-        long sizeInBytes
+        long sizeInBytes,
+        boolean hasCustomCutoffDate
     ) {
         super(XPackField.LOGSDB, available, enabled);
         this.indicesCount = indicesCount;
         this.indicesWithSyntheticSource = indicesWithSyntheticSource;
         this.numDocs = numDocs;
         this.sizeInBytes = sizeInBytes;
+        this.hasCustomCutoffDate = hasCustomCutoffDate;
     }
 
     @Override
@@ -74,11 +89,12 @@ protected void innerXContent(XContentBuilder builder, Params params) throws IOEx
         builder.field("indices_with_synthetic_source", indicesWithSyntheticSource);
         builder.field("num_docs", numDocs);
         builder.field("size_in_bytes", sizeInBytes);
+        builder.field("has_custom_cutoff_date", hasCustomCutoffDate);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(available, enabled, indicesCount, indicesWithSyntheticSource, numDocs, sizeInBytes);
+        return Objects.hash(available, enabled, indicesCount, indicesWithSyntheticSource, numDocs, sizeInBytes, hasCustomCutoffDate);
     }
 
     @Override
@@ -95,6 +111,7 @@ public boolean equals(Object obj) {
             && Objects.equals(indicesCount, other.indicesCount)
             && Objects.equals(indicesWithSyntheticSource, other.indicesWithSyntheticSource)
             && Objects.equals(numDocs, other.numDocs)
-            && Objects.equals(sizeInBytes, other.sizeInBytes);
+            && Objects.equals(sizeInBytes, other.sizeInBytes)
+            && Objects.equals(hasCustomCutoffDate, other.hasCustomCutoffDate);
     }
 }
diff --git a/x-pack/plugin/logsdb/qa/with-custom-cutoff/build.gradle b/x-pack/plugin/logsdb/qa/with-custom-cutoff/build.gradle
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+apply plugin: 'elasticsearch.internal-java-rest-test'
+
+dependencies {
+  javaRestTestImplementation(testArtifact(project(xpackModule('core'))))
+}
+
+tasks.named("javaRestTest").configure {
+  // This test cluster is using a BASIC license and FIPS 140 mode is not supported in BASIC
+  buildParams.withFipsEnabledOnly(it)
+
+  usesDefaultDistribution()
+}
diff --git a/x-pack/plugin/logsdb/qa/with-custom-cutoff/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbWithBasicRestIT.java b/x-pack/plugin/logsdb/qa/with-custom-cutoff/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/LogsdbWithBasicRestIT.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.logsdb;
+
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.hamcrest.Matchers;
+import org.junit.ClassRule;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class LogsdbWithBasicRestIT extends ESRestTestCase {
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .systemProperty("es.mapping.synthetic_source_fallback_to_stored_source.cutoff_date_restricted_override", "2027-12-31T23:59")
+        .setting("xpack.security.enabled", "false")
+        .setting("cluster.logsdb.enabled", "true")
+        .build();
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+
+    public void testCustomCutoffDateUsage() throws IOException {
+        var response = getAsMap("/_xpack/usage");
+        Map<?, ?> usage = (Map<?, ?>) response.get("logsdb");
+        assertThat(usage, Matchers.hasEntry("available", true));
+        assertThat(usage, Matchers.hasEntry("enabled", true));
+        assertThat(usage, Matchers.hasEntry("indices_count", 0));
+        assertThat(usage, Matchers.hasEntry("indices_with_synthetic_source", 0));
+        assertThat(usage, Matchers.hasEntry("num_docs", 0));
+        assertThat(usage, Matchers.hasEntry("size_in_bytes", 0));
+        assertThat(usage, Matchers.hasEntry("has_custom_cutoff_date", true));
+    }
+}
diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBUsageTransportAction.java
@@ -77,6 +77,7 @@ protected void masterOperation(
             }
         }
         final boolean enabled = LogsDBPlugin.CLUSTER_LOGSDB_ENABLED.get(clusterService.getSettings());
+        final boolean hasCustomCutoffDate = System.getProperty(SyntheticSourceLicenseService.CUTOFF_DATE_SYS_PROP_NAME) != null;
         if (featureService.clusterHasFeature(state, XPackFeatures.LOGSDB_TELMETRY_STATS)) {
             final DiscoveryNode[] nodes = state.nodes().getDataNodes().values().toArray(DiscoveryNode[]::new);
             final var statsRequest = new IndexModeStatsActionType.StatsRequest(nodes);
@@ -91,13 +92,16 @@ protected void masterOperation(
                         finalNumIndices,
                         finalNumIndicesWithSyntheticSources,
                         indexStats.numDocs(),
-                        indexStats.numBytes()
+                        indexStats.numBytes(),
+                        hasCustomCutoffDate
                     )
                 );
             }));
         } else {
             listener.onResponse(
-                new XPackUsageFeatureResponse(new LogsDBFeatureSetUsage(true, enabled, numIndices, numIndicesWithSyntheticSources, 0L, 0L))
+                new XPackUsageFeatureResponse(
+                    new LogsDBFeatureSetUsage(true, enabled, numIndices, numIndicesWithSyntheticSources, 0L, 0L, hasCustomCutoffDate)
+                )
             );
         }
     }
diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceLicenseService.java
@@ -27,8 +27,7 @@ final class SyntheticSourceLicenseService {
 
     static final String MAPPINGS_FEATURE_FAMILY = "mappings";
     // You can only override this property if you received explicit approval from Elastic.
-    private static final String CUTOFF_DATE_SYS_PROP_NAME =
-        "es.mapping.synthetic_source_fallback_to_stored_source.cutoff_date_restricted_override";
+    static final String CUTOFF_DATE_SYS_PROP_NAME = "es.mapping.synthetic_source_fallback_to_stored_source.cutoff_date_restricted_override";
     private static final Logger LOGGER = LogManager.getLogger(SyntheticSourceLicenseService.class);
     static final long DEFAULT_CUTOFF_DATE = LocalDateTime.of(2024, 12, 12, 0, 0).toInstant(ZoneOffset.UTC).toEpochMilli();
 
@@ -129,7 +128,7 @@ private static long getCutoffDate(String cutoffDateAsString) {
             LOGGER.info(
                 "Configuring [{}] to [{}]",
                 CUTOFF_DATE_SYS_PROP_NAME,
-                LocalDateTime.ofInstant(Instant.ofEpochSecond(cutoffDate), ZoneOffset.UTC)
+                LocalDateTime.ofInstant(Instant.ofEpochMilli(cutoffDate), ZoneOffset.UTC)
             );
             return cutoffDate;
         } else {

Original file line number	Diff line number	Diff line change
`@@ -77,6 +77,7 @@ protected void masterOperation(`
`77`	`77`	`}`
`78`	`78`	`}`
`79`	`79`	`final boolean enabled = LogsDBPlugin.CLUSTER_LOGSDB_ENABLED.get(clusterService.getSettings());`
	`80`	`+ final boolean hasCustomCutoffDate = System.getProperty(SyntheticSourceLicenseService.CUTOFF_DATE_SYS_PROP_NAME) != null;`
`80`	`81`	`if (featureService.clusterHasFeature(state, XPackFeatures.LOGSDB_TELMETRY_STATS)) {`
`81`	`82`	`final DiscoveryNode[] nodes = state.nodes().getDataNodes().values().toArray(DiscoveryNode[]::new);`
`82`	`83`	`final var statsRequest = new IndexModeStatsActionType.StatsRequest(nodes);`
`@@ -91,13 +92,16 @@ protected void masterOperation(`
`91`	`92`	`finalNumIndices,`
`92`	`93`	`finalNumIndicesWithSyntheticSources,`
`93`	`94`	`indexStats.numDocs(),`
`94`		`- indexStats.numBytes()`
	`95`	`+ indexStats.numBytes(),`
	`96`	`+ hasCustomCutoffDate`
`95`	`97`	`)`
`96`	`98`	`);`
`97`	`99`	`}));`
`98`	`100`	`} else {`
`99`	`101`	`listener.onResponse(`
`100`		`- new XPackUsageFeatureResponse(new LogsDBFeatureSetUsage(true, enabled, numIndices, numIndicesWithSyntheticSources, 0L, 0L))`
	`102`	`+ new XPackUsageFeatureResponse(`
	`103`	`+ new LogsDBFeatureSetUsage(true, enabled, numIndices, numIndicesWithSyntheticSources, 0L, 0L, hasCustomCutoffDate)`
	`104`	`+ )`
`101`	`105`	`);`
`102`	`106`	`}`
`103`	`107`	`}`