Skip to content

Commit 4eabb4c

Browse files
parkertimminsparkertimmins
authored
Add rest endpoint for create_from_source_index (#119250) (#119777)
Add rest endpoint which creates a new index using the settings and mappings from an existing source index. Setting and mapping overrides can be added to the request which will override settings and mappings from the source index. Example: `PUT /_create_from/{source}/{dest}` Content is optional but can include a `settings_override` and `mappings_override` which will be combined with the settings and mappings from the source index to create the destination index.
1 parent 43e4404 commit 4eabb4c

File tree

9 files changed

+367
-15
lines changed

9 files changed

+367
-15
lines changed

‎docs/changelog/119250.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 119250
2+
summary: Add rest endpoint for `create_from_source_index`
3+
area: Data streams
4+
type: enhancement
5+
issues: []

‎rest-api-spec/src/main/resources/rest-api-spec/api/migrate.create_from.json‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
{
2+
"migrate.create_from":{
3+
"documentation":{
4+
"url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-stream-reindex.html",
5+
"description":"This API creates a destination from a source index. It copies the mappings and settings from the source index while allowing request settings and mappings to override the source values."
6+
},
7+
"stability":"experimental",
8+
"visibility":"private",
9+
"headers":{
10+
"accept": [ "application/json"],
11+
"content_type": ["application/json"]
12+
},
13+
"url":{
14+
"paths":[
15+
{
16+
"path":"/_create_from/{source}/{dest}",
17+
"methods":[ "PUT", "POST"],
18+
"parts":{
19+
"source":{
20+
"type":"string",
21+
"description":"The source index name"
22+
},
23+
"dest":{
24+
"type":"string",
25+
"description":"The destination index name"
26+
}
27+
}
28+
}
29+
]
30+
},
31+
"body":{
32+
"description":"The body contains the fields `mappings_override` and `settings_override`.",
33+
"required":false
34+
}
35+
}
36+
}
37+

‎x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
import org.elasticsearch.xpack.migrate.action.ReindexDataStreamIndexTransportAction;
4444
import org.elasticsearch.xpack.migrate.action.ReindexDataStreamTransportAction;
4545
import org.elasticsearch.xpack.migrate.rest.RestCancelReindexDataStreamAction;
46+
import org.elasticsearch.xpack.migrate.rest.RestCreateIndexFromSourceAction;
4647
import org.elasticsearch.xpack.migrate.rest.RestGetMigrationReindexStatusAction;
4748
import org.elasticsearch.xpack.migrate.rest.RestMigrationReindexAction;
4849
import org.elasticsearch.xpack.migrate.task.ReindexDataStreamPersistentTaskExecutor;
@@ -77,6 +78,7 @@ public List<RestHandler> getRestHandlers(
7778
handlers.add(new RestMigrationReindexAction());
7879
handlers.add(new RestGetMigrationReindexStatusAction());
7980
handlers.add(new RestCancelReindexDataStreamAction());
81+
handlers.add(new RestCreateIndexFromSourceAction());
8082
}
8183
return handlers;
8284
}

‎x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceAction.java‎

Lines changed: 60 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,11 @@
1515
import org.elasticsearch.common.io.stream.StreamInput;
1616
import org.elasticsearch.common.io.stream.StreamOutput;
1717
import org.elasticsearch.common.settings.Settings;
18+
import org.elasticsearch.xcontent.ObjectParser;
19+
import org.elasticsearch.xcontent.ParseField;
20+
import org.elasticsearch.xcontent.ToXContent;
21+
import org.elasticsearch.xcontent.XContentBuilder;
22+
import org.elasticsearch.xcontent.XContentParser;
1823

1924
import java.io.IOException;
2025
import java.util.Map;
@@ -30,18 +35,35 @@ private CreateIndexFromSourceAction() {
3035
super(NAME);
3136
}
3237

33-
public static class Request extends ActionRequest implements IndicesRequest {
34-
38+
public static class Request extends ActionRequest implements IndicesRequest, ToXContent {
3539
private final String sourceIndex;
3640
private final String destIndex;
37-
private final Settings settingsOverride;
38-
private final Map<String, Object> mappingsOverride;
41+
private Settings settingsOverride = Settings.EMPTY;
42+
private Map<String, Object> mappingsOverride = Map.of();
43+
private static final ParseField SETTINGS_OVERRIDE_FIELD = new ParseField("settings_override");
44+
private static final ParseField MAPPINGS_OVERRIDE_FIELD = new ParseField("mappings_override");
45+
private static final ObjectParser<Request, Void> PARSER = new ObjectParser<>("create_index_from_source_request");
46+
47+
static {
48+
PARSER.declareField(
49+
(parser, request, context) -> request.settingsOverride(Settings.fromXContent(parser)),
50+
SETTINGS_OVERRIDE_FIELD,
51+
ObjectParser.ValueType.OBJECT
52+
);
53+
54+
PARSER.declareField(
55+
(parser, request, context) -> request.mappingsOverride(Map.of("_doc", parser.map())),
56+
MAPPINGS_OVERRIDE_FIELD,
57+
ObjectParser.ValueType.OBJECT
58+
);
59+
}
3960

4061
public Request(String sourceIndex, String destIndex) {
4162
this(sourceIndex, destIndex, Settings.EMPTY, Map.of());
4263
}
4364

4465
public Request(String sourceIndex, String destIndex, Settings settingsOverride, Map<String, Object> mappingsOverride) {
66+
Objects.requireNonNull(settingsOverride);
4567
Objects.requireNonNull(mappingsOverride);
4668
this.sourceIndex = sourceIndex;
4769
this.destIndex = destIndex;
@@ -72,22 +94,52 @@ public ActionRequestValidationException validate() {
7294
return null;
7395
}
7496

75-
public String getSourceIndex() {
97+
public String sourceIndex() {
7698
return sourceIndex;
7799
}
78100

79-
public String getDestIndex() {
101+
public String destIndex() {
80102
return destIndex;
81103
}
82104

83-
public Settings getSettingsOverride() {
105+
public Settings settingsOverride() {
84106
return settingsOverride;
85107
}
86108

87-
public Map<String, Object> getMappingsOverride() {
109+
public Map<String, Object> mappingsOverride() {
88110
return mappingsOverride;
89111
}
90112

113+
public void settingsOverride(Settings settingsOverride) {
114+
this.settingsOverride = settingsOverride;
115+
}
116+
117+
public void mappingsOverride(Map<String, Object> mappingsOverride) {
118+
this.mappingsOverride = mappingsOverride;
119+
}
120+
121+
public void fromXContent(XContentParser parser) throws IOException {
122+
PARSER.parse(parser, this, null);
123+
}
124+
125+
/*
126+
* This only exists for the sake of testing the xcontent parser
127+
*/
128+
@Override
129+
public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
130+
if (mappingsOverride.containsKey("_doc")) {
131+
builder.field(MAPPINGS_OVERRIDE_FIELD.getPreferredName(), mappingsOverride.get("_doc"));
132+
}
133+
134+
if (settingsOverride.isEmpty() == false) {
135+
builder.startObject(SETTINGS_OVERRIDE_FIELD.getPreferredName());
136+
settingsOverride.toXContent(builder, params);
137+
builder.endObject();
138+
}
139+
140+
return builder;
141+
}
142+
91143
@Override
92144
public boolean equals(Object o) {
93145
if (this == o) return true;

‎x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceTransportAction.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -69,31 +69,31 @@ public CreateIndexFromSourceTransportAction(
6969
@Override
7070
protected void doExecute(Task task, CreateIndexFromSourceAction.Request request, ActionListener<AcknowledgedResponse> listener) {
7171

72-
IndexMetadata sourceIndex = clusterService.state().getMetadata().index(request.getSourceIndex());
72+
IndexMetadata sourceIndex = clusterService.state().getMetadata().index(request.sourceIndex());
7373

7474
if (sourceIndex == null) {
75-
listener.onFailure(new IndexNotFoundException(request.getSourceIndex()));
75+
listener.onFailure(new IndexNotFoundException(request.sourceIndex()));
7676
return;
7777
}
7878

79-
logger.debug("Creating destination index [{}] for source index [{}]", request.getDestIndex(), request.getSourceIndex());
79+
logger.debug("Creating destination index [{}] for source index [{}]", request.destIndex(), request.sourceIndex());
8080

8181
Settings settings = Settings.builder()
8282
// add source settings
8383
.put(filterSettings(sourceIndex))
8484
// add override settings from request
85-
.put(request.getSettingsOverride())
85+
.put(request.settingsOverride())
8686
.build();
8787

8888
Map<String, Object> mergeMappings;
8989
try {
90-
mergeMappings = mergeMappings(sourceIndex.mapping(), request.getMappingsOverride());
90+
mergeMappings = mergeMappings(sourceIndex.mapping(), request.mappingsOverride());
9191
} catch (IOException e) {
9292
listener.onFailure(e);
9393
return;
9494
}
9595

96-
var createIndexRequest = new CreateIndexRequest(request.getDestIndex()).settings(settings);
96+
var createIndexRequest = new CreateIndexRequest(request.destIndex()).settings(settings);
9797
if (mergeMappings.isEmpty() == false) {
9898
createIndexRequest.mapping(mergeMappings);
9999
}

‎x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ private void createIndex(
156156
Map.of()
157157
);
158158
request.setParentTask(parentTaskId);
159-
var errorMessage = String.format(Locale.ROOT, "Could not create index [%s]", request.getDestIndex());
159+
var errorMessage = String.format(Locale.ROOT, "Could not create index [%s]", request.destIndex());
160160
client.execute(CreateIndexFromSourceAction.INSTANCE, request, failIfNotAcknowledged(listener, errorMessage));
161161
}
162162

‎x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/rest/RestCreateIndexFromSourceAction.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.migrate.rest;
9+
10+
import org.elasticsearch.client.internal.node.NodeClient;
11+
import org.elasticsearch.rest.BaseRestHandler;
12+
import org.elasticsearch.rest.RestRequest;
13+
import org.elasticsearch.rest.action.RestToXContentListener;
14+
import org.elasticsearch.xpack.migrate.action.CreateIndexFromSourceAction;
15+
16+
import java.io.IOException;
17+
import java.util.List;
18+
19+
import static org.elasticsearch.rest.RestRequest.Method.POST;
20+
import static org.elasticsearch.rest.RestRequest.Method.PUT;
21+
22+
public class RestCreateIndexFromSourceAction extends BaseRestHandler {
23+
24+
@Override
25+
public String getName() {
26+
return "create_index_from_source_action";
27+
}
28+
29+
@Override
30+
public List<Route> routes() {
31+
return List.of(new Route(PUT, "/_create_from/{source}/{dest}"), new Route(POST, "/_create_from/{source}/{dest}"));
32+
}
33+
34+
@Override
35+
protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
36+
var createRequest = new CreateIndexFromSourceAction.Request(request.param("source"), request.param("dest"));
37+
request.applyContentParser(createRequest::fromXContent);
38+
return channel -> client.execute(CreateIndexFromSourceAction.INSTANCE, createRequest, new RestToXContentListener<>(channel));
39+
}
40+
}

‎x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/CreateFromSourceIndexRequestTests.java‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.migrate.action;
9+
10+
import org.elasticsearch.common.bytes.BytesReference;
11+
import org.elasticsearch.common.io.stream.Writeable;
12+
import org.elasticsearch.common.settings.Settings;
13+
import org.elasticsearch.test.AbstractWireSerializingTestCase;
14+
import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
15+
import org.elasticsearch.xcontent.XContentParser;
16+
import org.elasticsearch.xcontent.XContentType;
17+
import org.elasticsearch.xpack.migrate.action.CreateIndexFromSourceAction.Request;
18+
19+
import java.io.IOException;
20+
import java.util.Map;
21+
22+
import static org.elasticsearch.xcontent.ToXContent.EMPTY_PARAMS;
23+
24+
public class CreateFromSourceIndexRequestTests extends AbstractWireSerializingTestCase<Request> {
25+
26+
public void testToAndFromXContent() throws IOException {
27+
var request = createTestInstance();
28+
29+
boolean humanReadable = randomBoolean();
30+
final XContentType xContentType = randomFrom(XContentType.values());
31+
BytesReference originalBytes = toShuffledXContent(request, xContentType, EMPTY_PARAMS, humanReadable);
32+
33+
var parsedRequest = new Request(
34+
randomValueOtherThan(request.sourceIndex(), () -> randomAlphanumericOfLength(30)),
35+
randomValueOtherThan(request.sourceIndex(), () -> randomAlphanumericOfLength(30))
36+
);
37+
try (XContentParser xParser = createParser(xContentType.xContent(), originalBytes)) {
38+
parsedRequest.fromXContent(xParser);
39+
}
40+
41+
// source and dest won't be equal
42+
assertNotEquals(request, parsedRequest);
43+
assertNotEquals(request.sourceIndex(), parsedRequest.sourceIndex());
44+
assertNotEquals(request.destIndex(), parsedRequest.destIndex());
45+
46+
// but fields in xcontent will be equal
47+
assertEquals(request.settingsOverride(), parsedRequest.settingsOverride());
48+
assertEquals(request.mappingsOverride(), parsedRequest.mappingsOverride());
49+
50+
BytesReference finalBytes = toShuffledXContent(parsedRequest, xContentType, EMPTY_PARAMS, humanReadable);
51+
ElasticsearchAssertions.assertToXContentEquivalent(originalBytes, finalBytes, xContentType);
52+
}
53+
54+
@Override
55+
protected Writeable.Reader<Request> instanceReader() {
56+
return Request::new;
57+
}
58+
59+
@Override
60+
protected Request createTestInstance() {
61+
String source = randomAlphaOfLength(30);
62+
String dest = randomAlphaOfLength(30);
63+
if (randomBoolean()) {
64+
return new Request(source, dest);
65+
} else {
66+
return new Request(source, dest, randomSettings(), randomMappings());
67+
}
68+
}
69+
70+
@Override
71+
protected Request mutateInstance(Request instance) throws IOException {
72+
73+
String sourceIndex = instance.sourceIndex();
74+
String destIndex = instance.destIndex();
75+
Settings settingsOverride = instance.settingsOverride();
76+
Map<String, Object> mappingsOverride = instance.mappingsOverride();
77+
78+
switch (between(0, 3)) {
79+
case 0 -> sourceIndex = randomValueOtherThan(sourceIndex, () -> randomAlphaOfLength(30));
80+
case 1 -> destIndex = randomValueOtherThan(destIndex, () -> randomAlphaOfLength(30));
81+
case 2 -> settingsOverride = randomValueOtherThan(settingsOverride, CreateFromSourceIndexRequestTests::randomSettings);
82+
case 3 -> mappingsOverride = randomValueOtherThan(mappingsOverride, CreateFromSourceIndexRequestTests::randomMappings);
83+
}
84+
return new Request(sourceIndex, destIndex, settingsOverride, mappingsOverride);
85+
}
86+
87+
public static Map<String, Object> randomMappings() {
88+
var randMappings = Map.of("properties", Map.of(randomAlphaOfLength(5), Map.of("type", "keyword")));
89+
return randomBoolean() ? Map.of() : Map.of("_doc", randMappings);
90+
}
91+
92+
public static Settings randomSettings() {
93+
return randomBoolean() ? Settings.EMPTY : indexSettings(randomIntBetween(1, 10), randomIntBetween(0, 5)).build();
94+
}
95+
}

0 commit comments

Comments
 (0)