Skip to content

Commit 24cda39

Browse files
szybiaszybia
committed
proposal
1 parent 11a2dfe commit 24cda39

File tree

6 files changed

+76
-36
lines changed

6 files changed

+76
-36
lines changed

‎rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/ingest/20_tracking.yml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,10 @@ setup:
2020

2121
- do:
2222
ingest.get_pipeline:
23+
human: true
2324
id: "my_pipeline"
25+
- gte: { my_pipeline.created_date_millis: 0 }
26+
- gte: { my_pipeline.modified_date_millis: 0 }
2427
- match: { my_pipeline.created_date: "/^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$/" }
2528
- match: { my_pipeline.modified_date: "/^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$/" }
2629

@@ -38,6 +41,34 @@ setup:
3841
- match: { status: 400 }
3942
- match: { error.reason: "Provided a pipeline property which is managed by the system: created_date." }
4043

44+
---
45+
"Test PUT setting created_date_millis":
46+
- do:
47+
catch: bad_request
48+
ingest.put_pipeline:
49+
id: "my_pipeline"
50+
body: >
51+
{
52+
"processors": [],
53+
"created_date_millis": 0
54+
}
55+
- match: { status: 400 }
56+
- match: { error.reason: "Provided a pipeline property which is managed by the system: created_date_millis." }
57+
58+
---
59+
"Test PUT setting modified_date_millis":
60+
- do:
61+
catch: bad_request
62+
ingest.put_pipeline:
63+
id: "my_pipeline"
64+
body: >
65+
{
66+
"processors": [],
67+
"modified_date_millis": 0
68+
}
69+
- match: { status: 400 }
70+
- match: { error.reason: "Provided a pipeline property which is managed by the system: modified_date_millis." }
71+
4172
---
4273
"Test PUT setting modified_date":
4374
- do:

‎server/src/main/java/org/elasticsearch/action/ingest/GetPipelineResponse.java‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.elasticsearch.common.Strings;
1414
import org.elasticsearch.common.io.stream.StreamOutput;
1515
import org.elasticsearch.core.UpdateForV10;
16+
import org.elasticsearch.ingest.Pipeline;
1617
import org.elasticsearch.ingest.PipelineConfiguration;
1718
import org.elasticsearch.rest.RestStatus;
1819
import org.elasticsearch.xcontent.ToXContentObject;
@@ -74,7 +75,25 @@ public RestStatus status() {
7475
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
7576
builder.startObject();
7677
for (PipelineConfiguration pipeline : pipelines) {
77-
builder.field(pipeline.getId(), summary ? Map.of() : pipeline.getConfig());
78+
builder.startObject(pipeline.getId());
79+
for (final Map.Entry<String, Object> configProperty : (summary ? Map.<String, Object>of() : pipeline.getConfig()).entrySet()) {
80+
if (Pipeline.CREATED_DATE_MILLIS.equals(configProperty.getKey())) {
81+
builder.timestampFieldsFromUnixEpochMillis(
82+
Pipeline.CREATED_DATE_MILLIS,
83+
Pipeline.CREATED_DATE,
84+
(Long) configProperty.getValue()
85+
);
86+
} else if (Pipeline.MODIFIED_DATE_MILLIS.equals(configProperty.getKey())) {
87+
builder.timestampFieldsFromUnixEpochMillis(
88+
Pipeline.MODIFIED_DATE_MILLIS,
89+
Pipeline.MODIFIED_DATE,
90+
(Long) configProperty.getValue()
91+
);
92+
} else {
93+
builder.field(configProperty.getKey(), configProperty.getValue());
94+
}
95+
}
96+
builder.endObject();
7897
}
7998
builder.endObject();
8099
return builder;

‎server/src/main/java/org/elasticsearch/ingest/IngestService.java‎

Lines changed: 13 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -80,9 +80,6 @@
8080

8181
import java.time.Instant;
8282
import java.time.InstantSource;
83-
import java.time.format.DateTimeFormatter;
84-
import java.time.format.DateTimeFormatterBuilder;
85-
import java.time.temporal.ChronoUnit;
8683
import java.util.ArrayList;
8784
import java.util.Collection;
8885
import java.util.Collections;
@@ -572,9 +569,13 @@ public void validatePipelineRequest(ProjectId projectId, PutPipelineRequest requ
572569
}
573570

574571
public static void validateNoSystemPropertiesInPipelineConfig(final Map<String, Object> pipelineConfig) {
575-
if (pipelineConfig.containsKey(Pipeline.CREATED_DATE_KEY)) {
572+
if (pipelineConfig.containsKey(Pipeline.CREATED_DATE_MILLIS)) {
573+
throw new ElasticsearchParseException("Provided a pipeline property which is managed by the system: created_date_millis.");
574+
} else if (pipelineConfig.containsKey(Pipeline.CREATED_DATE)) {
576575
throw new ElasticsearchParseException("Provided a pipeline property which is managed by the system: created_date.");
577-
} else if (pipelineConfig.containsKey(Pipeline.MODIFIED_DATE_KEY)) {
576+
} else if (pipelineConfig.containsKey(Pipeline.MODIFIED_DATE_MILLIS)) {
577+
throw new ElasticsearchParseException("Provided a pipeline property which is managed by the system: modified_date_millis.");
578+
} else if (pipelineConfig.containsKey(Pipeline.MODIFIED_DATE)) {
578579
throw new ElasticsearchParseException("Provided a pipeline property which is managed by the system: modified_date.");
579580
}
580581
}
@@ -593,8 +594,8 @@ public static boolean isNoOpPipelineUpdate(ProjectMetadata metadata, PutPipeline
593594
Map<String, Object> currentConfigWithoutSystemProps = new HashMap<>(
594595
currentIngestMetadata.getPipelines().get(request.getId()).getConfig()
595596
);
596-
currentConfigWithoutSystemProps.remove(Pipeline.CREATED_DATE_KEY);
597-
currentConfigWithoutSystemProps.remove(Pipeline.MODIFIED_DATE_KEY);
597+
currentConfigWithoutSystemProps.remove(Pipeline.CREATED_DATE_MILLIS);
598+
currentConfigWithoutSystemProps.remove(Pipeline.MODIFIED_DATE_MILLIS);
598599

599600
return newPipelineConfig.equals(currentConfigWithoutSystemProps);
600601
}
@@ -693,10 +694,6 @@ private static void collectProcessorMetrics(
693694
* Used in this class and externally by the {@link org.elasticsearch.action.ingest.ReservedPipelineAction}
694695
*/
695696
public static class PutPipelineClusterStateUpdateTask extends PipelineClusterStateUpdateTask {
696-
// always output millis even if instantSource returns millis == 0
697-
private static final DateTimeFormatter ISO8601_WITH_MILLIS_FORMATTER = new DateTimeFormatterBuilder().appendInstant(3)
698-
.toFormatter(Locale.ROOT);
699-
700697
private final PutPipelineRequest request;
701698
private final InstantSource instantSource;
702699

@@ -779,20 +776,18 @@ public IngestMetadata execute(IngestMetadata currentIngestMetadata, Collection<I
779776
}
780777
}
781778

782-
final String iso8601WithMillisNow = ISO8601_WITH_MILLIS_FORMATTER.format(
783-
instantSource.instant().truncatedTo(ChronoUnit.MILLIS)
784-
);
779+
final long nowMillis = instantSource.millis();
785780
if (existingPipeline == null) {
786-
newPipelineConfig.put(Pipeline.CREATED_DATE_KEY, iso8601WithMillisNow);
781+
newPipelineConfig.put(Pipeline.CREATED_DATE_MILLIS, nowMillis);
787782
} else {
788-
Object existingCreatedAt = existingPipeline.getConfig().get(Pipeline.CREATED_DATE_KEY);
783+
Object existingCreatedAt = existingPipeline.getConfig().get(Pipeline.CREATED_DATE_MILLIS);
789784
// only set/carry over `created_date` if existing pipeline already has it.
790785
// would be confusing if existing pipelines were all updated to have `created_date` set to now.
791786
if (existingCreatedAt != null) {
792-
newPipelineConfig.put(Pipeline.CREATED_DATE_KEY, existingCreatedAt);
787+
newPipelineConfig.put(Pipeline.CREATED_DATE_MILLIS, existingCreatedAt);
793788
}
794789
}
795-
newPipelineConfig.put(Pipeline.MODIFIED_DATE_KEY, iso8601WithMillisNow);
790+
newPipelineConfig.put(Pipeline.MODIFIED_DATE_MILLIS, nowMillis);
796791

797792
pipelines.put(request.getId(), new PipelineConfiguration(request.getId(), newPipelineConfig));
798793
return new IngestMetadata(pipelines);

‎server/src/main/java/org/elasticsearch/ingest/Pipeline.java‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
import org.elasticsearch.features.NodeFeature;
1616
import org.elasticsearch.script.ScriptService;
1717

18-
import java.time.Instant;
1918
import java.util.Arrays;
2019
import java.util.List;
2120
import java.util.Map;
@@ -36,8 +35,10 @@ public final class Pipeline {
3635
public static final String META_KEY = "_meta";
3736
public static final String FIELD_ACCESS_PATTERN = "field_access_pattern";
3837
public static final String DEPRECATED_KEY = "deprecated";
39-
public static final String CREATED_DATE_KEY = "created_date";
40-
public static final String MODIFIED_DATE_KEY = "modified_date";
38+
public static final String CREATED_DATE_MILLIS = "created_date_millis";
39+
public static final String CREATED_DATE = "created_date";
40+
public static final String MODIFIED_DATE_MILLIS = "modified_date_millis";
41+
public static final String MODIFIED_DATE = "modified_date";
4142

4243
private final String id;
4344
@Nullable
@@ -172,8 +173,8 @@ public static Pipeline create(
172173
processorFactories,
173174
projectId
174175
);
175-
String createdDate = ConfigurationUtils.readOptionalStringOrLongProperty(null, null, config, CREATED_DATE_KEY);
176-
String modifiedDate = ConfigurationUtils.readOptionalStringOrLongProperty(null, null, config, MODIFIED_DATE_KEY);
176+
String createdDate = ConfigurationUtils.readOptionalStringOrLongProperty(null, null, config, CREATED_DATE_MILLIS);
177+
String modifiedDate = ConfigurationUtils.readOptionalStringOrLongProperty(null, null, config, MODIFIED_DATE_MILLIS);
177178
if (config.isEmpty() == false) {
178179
throw new ElasticsearchParseException(
179180
"pipeline ["
@@ -186,8 +187,8 @@ public static Pipeline create(
186187
throw new ElasticsearchParseException("pipeline [" + id + "] cannot have an empty on_failure option defined");
187188
}
188189
CompoundProcessor compoundProcessor = new CompoundProcessor(false, processors, onFailureProcessors);
189-
Long createdDateMillis = createdDate == null ? null : Instant.parse(createdDate).toEpochMilli();
190-
Long modifiedDateMillis = modifiedDate == null ? null : Instant.parse(modifiedDate).toEpochMilli();
190+
Long createdDateMillis = createdDate == null ? null : Long.valueOf(createdDate);
191+
Long modifiedDateMillis = modifiedDate == null ? null : Long.valueOf(modifiedDate);
191192
return new Pipeline(
192193
id,
193194
description,

‎server/src/main/java/org/elasticsearch/ingest/PipelineConfiguration.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -253,15 +253,15 @@ PipelineConfiguration maybeUpgradeProcessors(String type, IngestMetadata.Process
253253

254254
private Map<String, Object> configForTransport(final TransportVersion transportVersion) {
255255
final boolean transportSupportsNewProperties = transportVersion.onOrAfter(TransportVersions.PIPELINE_TRACKING_INFO);
256-
final boolean noNewProperties = config.containsKey(Pipeline.CREATED_DATE_KEY) == false
257-
&& config.containsKey(Pipeline.MODIFIED_DATE_KEY) == false;
256+
final boolean noNewProperties = config.containsKey(Pipeline.CREATED_DATE_MILLIS) == false
257+
&& config.containsKey(Pipeline.MODIFIED_DATE_MILLIS) == false;
258258

259259
if (transportSupportsNewProperties || noNewProperties) {
260260
return config;
261261
}
262262
final Map<String, Object> configWithoutNewSystemProperties = new HashMap<>(config);
263-
configWithoutNewSystemProperties.remove(Pipeline.CREATED_DATE_KEY);
264-
configWithoutNewSystemProperties.remove(Pipeline.MODIFIED_DATE_KEY);
263+
configWithoutNewSystemProperties.remove(Pipeline.CREATED_DATE_MILLIS);
264+
configWithoutNewSystemProperties.remove(Pipeline.MODIFIED_DATE_MILLIS);
265265
return Collections.unmodifiableMap(configWithoutNewSystemProperties);
266266
}
267267
}

‎server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -834,12 +834,6 @@ public void testPutWithTracking() {
834834
Pipeline pipeline = ingestService.getPipeline(projectId, id);
835835
assertThat(pipeline.getCreatedDateMillis().orElseThrow(), is(0L));
836836
assertThat(pipeline.getModifiedDateMillis().orElseThrow(), is(0L));
837-
// test PipelineConfiguration has millis part in iso8601 even if instantSource provides millis part == 0
838-
final Map<String, Object> clusterStatePipelineConfig = ((IngestMetadata) clusterState.metadata()
839-
.getProject(projectId)
840-
.custom(IngestMetadata.TYPE)).getPipelines().get(id).getConfig();
841-
assertThat(clusterStatePipelineConfig.get(Pipeline.CREATED_DATE_KEY), is("1970-01-01T00:00:00.000Z"));
842-
assertThat(clusterStatePipelineConfig.get(Pipeline.MODIFIED_DATE_KEY), is("1970-01-01T00:00:00.000Z"));
843837

844838
// overwrite existing pipeline:
845839
putRequest = putJsonPipelineRequest(id, """

0 commit comments

Comments
 (0)