diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/sampling/RestPutSampleConfigurationAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/sampling/RestPutSampleConfigurationAction.java index be1e1a1c85727..df4f89ca520c2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/sampling/RestPutSampleConfigurationAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/sampling/RestPutSampleConfigurationAction.java @@ -71,7 +71,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC PutSampleConfigurationAction.Request putRequest; XContentParser parser = request.contentParser(); - SamplingConfiguration samplingConfig = SamplingConfiguration.fromXContent(parser); + SamplingConfiguration samplingConfig = SamplingConfiguration.fromXContentUserData(parser); putRequest = new PutSampleConfigurationAction.Request(samplingConfig, getMasterNodeTimeout(request), getAckTimeout(request)); // Set the target index diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/sampling/SamplingConfiguration.java b/server/src/main/java/org/elasticsearch/action/admin/indices/sampling/SamplingConfiguration.java index bb70f0bdf3786..2acc016cab0be 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/sampling/SamplingConfiguration.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/sampling/SamplingConfiguration.java @@ -23,6 +23,9 @@ import org.elasticsearch.xcontent.XContentParser; import java.io.IOException; +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; @@ -30,10 +33,14 @@ /** * Configuration for sampling raw documents in an index. */ -public record SamplingConfiguration(double rate, Integer maxSamples, ByteSizeValue maxSize, TimeValue timeToLive, String condition) - implements - ToXContentObject, - SimpleDiffable { +public record SamplingConfiguration( + double rate, + Integer maxSamples, + ByteSizeValue maxSize, + TimeValue timeToLive, + String condition, + Long creationTime +) implements ToXContentObject, SimpleDiffable { public static final String TYPE = "sampling_configuration"; private static final String RATE_FIELD_NAME = "rate"; @@ -43,6 +50,10 @@ public record SamplingConfiguration(double rate, Integer maxSamples, ByteSizeVal private static final String TIME_TO_LIVE_IN_MILLIS_FIELD_NAME = "time_to_live_in_millis"; private static final String TIME_TO_LIVE_FIELD_NAME = "time_to_live"; private static final String CONDITION_FIELD_NAME = "if"; + private static final String CREATION_TIME_IN_MILLIS_FIELD_NAME = "creation_time_in_millis"; + private static final String CREATION_TIME_FIELD_NAME = "creation_time"; + + private static final String IS_USER_DATA_CONTEXT_KEY = "is_user_data"; // Constants for validation and defaults public static final int MAX_SAMPLES_LIMIT = 10_000; @@ -64,10 +75,9 @@ public record SamplingConfiguration(double rate, Integer maxSamples, ByteSizeVal + " days"; public static final String INVALID_CONDITION_MESSAGE = "condition script, if provided, must not be empty"; - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - TYPE, - false, - args -> { + private static final ConstructingObjectParser> PARSER = new ConstructingObjectParser< + SamplingConfiguration, + Map>(TYPE, false, (args, context) -> { Double rate = (Double) args[0]; Integer maxSamples = (Integer) args[1]; ByteSizeValue humanReadableMaxSize = (ByteSizeValue) args[2]; @@ -75,16 +85,27 @@ public record SamplingConfiguration(double rate, Integer maxSamples, ByteSizeVal TimeValue humanReadableTimeToLive = (TimeValue) args[4]; TimeValue rawTimeToLive = (TimeValue) args[5]; String condition = (String) args[6]; + Long rawCreationTime = (Long) args[8]; + + if (context.get(IS_USER_DATA_CONTEXT_KEY)) { + validateInputs( + rate, + maxSamples, + determineValue(humanReadableMaxSize, rawMaxSize), + determineValue(humanReadableTimeToLive, rawTimeToLive), + condition + ); + } return new SamplingConfiguration( rate, maxSamples, determineValue(humanReadableMaxSize, rawMaxSize), determineValue(humanReadableTimeToLive, rawTimeToLive), - condition + condition, + rawCreationTime ); - } - ); + }); static { PARSER.declareDouble(constructorArg(), new ParseField(RATE_FIELD_NAME)); @@ -116,10 +137,18 @@ public record SamplingConfiguration(double rate, Integer maxSamples, ByteSizeVal ObjectParser.ValueType.LONG ); PARSER.declareString(optionalConstructorArg(), new ParseField(CONDITION_FIELD_NAME)); + PARSER.declareField(optionalConstructorArg(), (p, c) -> { + validateUserDataContext(c, CREATION_TIME_FIELD_NAME); + return Instant.parse(p.text()).toEpochMilli(); + }, new ParseField(CREATION_TIME_FIELD_NAME), ObjectParser.ValueType.STRING); + PARSER.declareField(optionalConstructorArg(), (p, c) -> { + validateUserDataContext(c, CREATION_TIME_IN_MILLIS_FIELD_NAME); + return p.longValue(); + }, new ParseField(CREATION_TIME_IN_MILLIS_FIELD_NAME), ObjectParser.ValueType.LONG); } /** - * Constructor with validation and defaulting for optional fields. + * Constructor with defaulting for optional fields. * * @param rate The fraction of documents to sample (must be between 0 and 1) * @param maxSamples The maximum number of documents to sample (optional, defaults to {@link #DEFAULT_MAX_SAMPLES}) @@ -129,15 +158,25 @@ public record SamplingConfiguration(double rate, Integer maxSamples, ByteSizeVal * @param condition An optional condition script that sampled documents must satisfy (optional, can be null) * @throws IllegalArgumentException If any of the parameters are invalid, according to the validation rules */ - public SamplingConfiguration(double rate, Integer maxSamples, ByteSizeValue maxSize, TimeValue timeToLive, String condition) { - validateInputs(rate, maxSamples, maxSize, timeToLive, condition); - - // Initialize record fields + public SamplingConfiguration( + double rate, + Integer maxSamples, + ByteSizeValue maxSize, + TimeValue timeToLive, + String condition, + Long creationTime + ) { this.rate = rate; this.maxSamples = maxSamples == null ? DEFAULT_MAX_SAMPLES : maxSamples; this.maxSize = maxSize == null ? ByteSizeValue.ofGb(DEFAULT_MAX_SIZE_GIGABYTES) : maxSize; this.timeToLive = timeToLive == null ? TimeValue.timeValueDays(DEFAULT_TIME_TO_LIVE_DAYS) : timeToLive; this.condition = condition; + this.creationTime = creationTime == null ? Instant.now().toEpochMilli() : creationTime; + } + + // Convenience constructor without creationTime + public SamplingConfiguration(double rate, Integer maxSamples, ByteSizeValue maxSize, TimeValue timeToLive, String condition) { + this(rate, maxSamples, maxSize, timeToLive, condition, null); } /** @@ -147,7 +186,7 @@ public SamplingConfiguration(double rate, Integer maxSamples, ByteSizeValue maxS * @throws IOException If an I/O error occurs during deserialization */ public SamplingConfiguration(StreamInput in) throws IOException { - this(in.readDouble(), in.readInt(), ByteSizeValue.readFrom(in), in.readTimeValue(), in.readOptionalString()); + this(in.readDouble(), in.readInt(), ByteSizeValue.readFrom(in), in.readTimeValue(), in.readOptionalString(), in.readLong()); } // Write to StreamOutput @@ -158,6 +197,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeWriteable(this.maxSize); out.writeTimeValue(this.timeToLive); out.writeOptionalString(this.condition); + out.writeLong(this.creationTime); } // Serialize to XContent (JSON) @@ -171,6 +211,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (condition != null) { builder.field(CONDITION_FIELD_NAME, condition); } + builder.timestampFieldsFromUnixEpochMillis(CREATION_TIME_IN_MILLIS_FIELD_NAME, CREATION_TIME_FIELD_NAME, creationTime); builder.endObject(); return builder; } @@ -183,7 +224,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws * @throws IOException If parsing fails due to invalid JSON or I/O errors */ public static SamplingConfiguration fromXContent(XContentParser parser) throws IOException { - return PARSER.parse(parser, null); + Map context = new HashMap<>(); + context.put(IS_USER_DATA_CONTEXT_KEY, false); + return PARSER.parse(parser, context); + } + + public static SamplingConfiguration fromXContentUserData(XContentParser parser) throws IOException { + return PARSER.parse(parser, Map.of(IS_USER_DATA_CONTEXT_KEY, true)); } /** @@ -249,4 +296,10 @@ private static T determineValue(T humanReadableValue, T rawValue) { return humanReadableValue != null ? humanReadableValue : rawValue; } + + private static void validateUserDataContext(Map context, String fieldName) { + if (context.get(IS_USER_DATA_CONTEXT_KEY) == Boolean.TRUE) { + throw new IllegalArgumentException("Creation time cannot be set by user (field: " + fieldName + ")"); + } + } } diff --git a/server/src/main/java/org/elasticsearch/ingest/SamplingService.java b/server/src/main/java/org/elasticsearch/ingest/SamplingService.java index 9a1ed81175570..d0a853ebf8a97 100644 --- a/server/src/main/java/org/elasticsearch/ingest/SamplingService.java +++ b/server/src/main/java/org/elasticsearch/ingest/SamplingService.java @@ -932,13 +932,14 @@ public Tuple executeTask( ) { logger.debug( "Updating sampling configuration for index [{}] with rate [{}]," - + " maxSamples [{}], maxSize [{}], timeToLive [{}], condition[{}]", + + " maxSamples [{}], maxSize [{}], timeToLive [{}], condition [{}], creationTime [{}]", updateSamplingConfigurationTask.indexName, updateSamplingConfigurationTask.samplingConfiguration.rate(), updateSamplingConfigurationTask.samplingConfiguration.maxSamples(), updateSamplingConfigurationTask.samplingConfiguration.maxSize(), updateSamplingConfigurationTask.samplingConfiguration.timeToLive(), - updateSamplingConfigurationTask.samplingConfiguration.condition() + updateSamplingConfigurationTask.samplingConfiguration.condition(), + updateSamplingConfigurationTask.samplingConfiguration.creationTime() ); // Get sampling metadata diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/sampling/SamplingConfigurationTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/sampling/SamplingConfigurationTests.java index b778605d0740b..f1be044f1c453 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/sampling/SamplingConfigurationTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/sampling/SamplingConfigurationTests.java @@ -17,10 +17,14 @@ import org.elasticsearch.xcontent.json.JsonXContent; import java.io.IOException; +import java.util.Locale; import java.util.Map; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.nullValue; +import static org.junit.Assert.assertNotNull; public class SamplingConfigurationTests extends AbstractXContentSerializingTestCase { @@ -102,78 +106,155 @@ public void testDefaults() { assertThat(config.condition(), nullValue()); } - public void testValidation() { + public void testValidation() throws IOException { // Test invalid rate - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> new SamplingConfiguration(-0.1, null, null, null, null) - ); - assertThat(e.getMessage(), equalTo(SamplingConfiguration.INVALID_RATE_MESSAGE)); - e = expectThrows(IllegalArgumentException.class, () -> new SamplingConfiguration(1.1, null, null, null, null)); - assertThat(e.getMessage(), equalTo(SamplingConfiguration.INVALID_RATE_MESSAGE)); + assertValidationError(""" + { + "rate": -0.1 + } + """, SamplingConfiguration.INVALID_RATE_MESSAGE); + + assertValidationError(""" + { + "rate": 1.1 + } + """, SamplingConfiguration.INVALID_RATE_MESSAGE); // Test invalid maxSamples - e = expectThrows(IllegalArgumentException.class, () -> new SamplingConfiguration(0.5, 0, null, null, null)); - assertThat(e.getMessage(), equalTo(SamplingConfiguration.INVALID_MAX_SAMPLES_MIN_MESSAGE)); - e = expectThrows(IllegalArgumentException.class, () -> new SamplingConfiguration(0.5, -1, null, null, null)); - assertThat(e.getMessage(), equalTo(SamplingConfiguration.INVALID_MAX_SAMPLES_MIN_MESSAGE)); - e = expectThrows( - IllegalArgumentException.class, - () -> new SamplingConfiguration(0.5, SamplingConfiguration.MAX_SAMPLES_LIMIT + 1, null, null, null) - ); - assertThat(e.getMessage(), equalTo(SamplingConfiguration.INVALID_MAX_SAMPLES_MAX_MESSAGE)); + assertValidationError(""" + { + "rate": 0.5, + "max_samples": 0 + } + """, SamplingConfiguration.INVALID_MAX_SAMPLES_MIN_MESSAGE); + + assertValidationError(""" + { + "rate": 0.5, + "max_samples": -1 + } + """, SamplingConfiguration.INVALID_MAX_SAMPLES_MIN_MESSAGE); + + assertValidationError(String.format(Locale.ROOT, """ + { + "rate": 0.5, + "max_samples": %d + } + """, SamplingConfiguration.MAX_SAMPLES_LIMIT + 1), SamplingConfiguration.INVALID_MAX_SAMPLES_MAX_MESSAGE); // Test invalid maxSize - e = expectThrows(IllegalArgumentException.class, () -> new SamplingConfiguration(0.5, null, ByteSizeValue.ZERO, null, null)); - assertThat(e.getMessage(), equalTo(SamplingConfiguration.INVALID_MAX_SIZE_MIN_MESSAGE)); - e = expectThrows(IllegalArgumentException.class, () -> new SamplingConfiguration(0.5, null, ByteSizeValue.ofBytes(-1), null, null)); - assertThat(e.getMessage(), equalTo(SamplingConfiguration.INVALID_MAX_SIZE_MIN_MESSAGE)); - e = expectThrows( - IllegalArgumentException.class, - () -> new SamplingConfiguration(0.5, null, ByteSizeValue.ofGb(SamplingConfiguration.MAX_SIZE_LIMIT_GIGABYTES + 1), null, null) - ); - assertThat(e.getMessage(), equalTo(SamplingConfiguration.INVALID_MAX_SIZE_MAX_MESSAGE)); + assertValidationError(""" + { + "rate": 0.5, + "max_size_in_bytes": 0 + } + """, SamplingConfiguration.INVALID_MAX_SIZE_MIN_MESSAGE); + + assertValidationError(""" + { + "rate": 0.5, + "max_size_in_bytes": -1 + } + """, SamplingConfiguration.INVALID_MAX_SIZE_MIN_MESSAGE); + + assertValidationError(String.format(Locale.ROOT, """ + { + "rate": 0.5, + "max_size": "%dgb" + } + """, SamplingConfiguration.MAX_SIZE_LIMIT_GIGABYTES + 1), SamplingConfiguration.INVALID_MAX_SIZE_MAX_MESSAGE); // Test invalid timeToLive - e = expectThrows(IllegalArgumentException.class, () -> new SamplingConfiguration(0.5, null, null, TimeValue.ZERO, null)); - assertThat(e.getMessage(), equalTo(SamplingConfiguration.INVALID_TIME_TO_LIVE_MIN_MESSAGE)); - e = expectThrows( - IllegalArgumentException.class, - () -> new SamplingConfiguration(0.5, null, null, TimeValue.timeValueDays(-1), null) - ); - assertThat(e.getMessage(), equalTo(SamplingConfiguration.INVALID_TIME_TO_LIVE_MIN_MESSAGE)); - e = expectThrows( - IllegalArgumentException.class, - () -> new SamplingConfiguration(0.5, null, null, TimeValue.timeValueDays(SamplingConfiguration.MAX_TIME_TO_LIVE_DAYS + 1), null) - ); - assertThat(e.getMessage(), equalTo(SamplingConfiguration.INVALID_TIME_TO_LIVE_MAX_MESSAGE)); + assertValidationError(""" + { + "rate": 0.5, + "time_to_live_in_millis": 0 + } + """, SamplingConfiguration.INVALID_TIME_TO_LIVE_MIN_MESSAGE); + + assertValidationError(""" + { + "rate": 0.5, + "time_to_live": "-1d" + } + """, SamplingConfiguration.INVALID_TIME_TO_LIVE_MIN_MESSAGE); + + assertValidationError(String.format(Locale.ROOT, """ + { + "rate": 0.5, + "time_to_live": "%dd" + } + """, SamplingConfiguration.MAX_TIME_TO_LIVE_DAYS + 1), SamplingConfiguration.INVALID_TIME_TO_LIVE_MAX_MESSAGE); // Test invalid condition - e = expectThrows(IllegalArgumentException.class, () -> new SamplingConfiguration(0.5, null, null, null, "")); - assertThat(e.getMessage(), equalTo(SamplingConfiguration.INVALID_CONDITION_MESSAGE)); + assertValidationError(""" + { + "rate": 0.5, + "if": "" + } + """, SamplingConfiguration.INVALID_CONDITION_MESSAGE); + } + // Helper method to find a cause with a specific message in the cause chain + private Throwable findCauseWithMessage(Throwable throwable, String expectedMessage) { + Throwable current = throwable; + while (current != null) { + if (current instanceof IllegalArgumentException && expectedMessage.equals(current.getMessage())) { + return current; + } + current = current.getCause(); + } + return null; } - public void testValidInputs() { - // Test boundary conditions - new SamplingConfiguration(0.001, 1, ByteSizeValue.ofBytes(1), TimeValue.timeValueMillis(1), randomAlphaOfLength(10)); // minimum - // values - new SamplingConfiguration( - 1.0, - SamplingConfiguration.MAX_SAMPLES_LIMIT, - ByteSizeValue.ofGb(SamplingConfiguration.MAX_SIZE_LIMIT_GIGABYTES), - TimeValue.timeValueDays(SamplingConfiguration.MAX_TIME_TO_LIVE_DAYS), - randomAlphaOfLength(10) - ); // maximum values - - // Test random valid values - new SamplingConfiguration( - randomDoubleBetween(0.0, 1.0, true), - randomIntBetween(1, SamplingConfiguration.MAX_SAMPLES_LIMIT), - ByteSizeValue.ofGb(randomLongBetween(1, SamplingConfiguration.MAX_SIZE_LIMIT_GIGABYTES)), - TimeValue.timeValueDays(randomLongBetween(1, SamplingConfiguration.MAX_TIME_TO_LIVE_DAYS)), - randomAlphaOfLength(10) + /** + * Helper method to test that fromXContentUserData throws validation errors for invalid JSON input + */ + private void assertValidationError(String jsonInput, String expectedErrorMessage) throws IOException { + XContentParser parser = createParser(JsonXContent.jsonXContent, jsonInput); + Exception e = expectThrows(Exception.class, () -> SamplingConfiguration.fromXContentUserData(parser)); + Throwable cause = findCauseWithMessage(e, expectedErrorMessage); + assertNotNull("Expected validation error: " + expectedErrorMessage, cause); + assertThat(cause.getMessage(), equalTo(expectedErrorMessage)); + } + + public void testValidInputs() throws IOException { + // Test boundary conditions - minimum values + XContentParser parser = createParser(JsonXContent.jsonXContent, """ + { + "rate": 0.001, + "max_samples": 1, + "max_size_in_bytes": 1, + "time_to_live_in_millis": 1, + "if": "test_condition" + } + """); + SamplingConfiguration config = SamplingConfiguration.fromXContent(parser); + assertThat(config.rate(), equalTo(0.001)); + assertThat(config.maxSamples(), equalTo(1)); + + // Test boundary conditions - maximum values + parser = createParser( + JsonXContent.jsonXContent, + String.format( + Locale.ROOT, + """ + { + "rate": 1.0, + "max_samples": %d, + "max_size": "%dgb", + "time_to_live": "%dd", + "if": "test_condition" + } + """, + SamplingConfiguration.MAX_SAMPLES_LIMIT, + SamplingConfiguration.MAX_SIZE_LIMIT_GIGABYTES, + SamplingConfiguration.MAX_TIME_TO_LIVE_DAYS + ) ); + config = SamplingConfiguration.fromXContent(parser); + assertThat(config.rate(), equalTo(1.0)); + assertThat(config.maxSamples(), equalTo(SamplingConfiguration.MAX_SAMPLES_LIMIT)); } @Override @@ -197,4 +278,58 @@ public void testHumanReadableParsing() throws IOException { assertThat(configuration.timeToLive(), equalTo(TimeValue.timeValueDays(1))); } + public void testCreationTime() throws IOException { + // Test that creation time is automatically set when not provided + long beforeCreation = java.time.Instant.now().toEpochMilli(); + SamplingConfiguration config = new SamplingConfiguration(0.5, null, null, null, null); + long afterCreation = java.time.Instant.now().toEpochMilli(); + + assertThat(config.creationTime(), greaterThanOrEqualTo(beforeCreation)); + assertThat(config.creationTime(), lessThanOrEqualTo(afterCreation)); + + // Test that explicit creation time is preserved + long explicitTime = java.time.Instant.parse("2023-10-05T12:34:56.789Z").toEpochMilli(); + SamplingConfiguration configWithTime = new SamplingConfiguration(0.5, null, null, null, null, explicitTime); + assertThat(configWithTime.creationTime(), equalTo(explicitTime)); + } + + public void testCreationTimeUserDataRestrictionHumanReadable() throws IOException { + // Test that user data cannot set creation time via human-readable field + final XContentParser parserA = createParser(JsonXContent.jsonXContent, """ + { + "rate": "0.05", + "creation_time": "2023-10-05T12:34:56.789Z" + } + """); + Exception e = expectThrows(Exception.class, () -> SamplingConfiguration.fromXContentUserData(parserA)); + // The IllegalArgumentException may be wrapped by the parser, so check the cause chain + Throwable cause = e.getCause(); + while (cause != null + && (cause instanceof IllegalArgumentException + && cause.getMessage().equals("Creation time cannot be set by user (field: creation_time)")) == false) { + cause = cause.getCause(); + } + assertNotNull("Expected IllegalArgumentException with creation_time message", cause); + assertThat(cause.getMessage(), equalTo("Creation time cannot be set by user (field: creation_time)")); + } + + public void testCreationTimeUserDataRestrictionRaw() throws IOException { + // Test that user data cannot set creation time via machine-readable field + final XContentParser parserB = createParser(JsonXContent.jsonXContent, """ + { + "rate": "0.05", + "creation_time_in_millis": 1696508096789 + } + """); + Exception e = expectThrows(Exception.class, () -> SamplingConfiguration.fromXContentUserData(parserB)); + // The IllegalArgumentException may be wrapped by the parser, so check the cause chain + Throwable cause = e; + while (cause != null + && (cause instanceof IllegalArgumentException + && cause.getMessage().equals("Creation time cannot be set by user (field: creation_time_in_millis)")) == false) { + cause = cause.getCause(); + } + assertNotNull("Expected IllegalArgumentException with creation_time_in_millis message", cause); + assertThat(cause.getMessage(), equalTo("Creation time cannot be set by user (field: creation_time_in_millis)")); + } }