Skip to content

Commit 10531df

Browse files
parkertimminsparkertimmins
authored
[8.x] Add action to create index from a source index (#118890) (#118990)
* Add action to create index from a source index (#118890) Add new action that creates an index from a source index, copying settings and mappings from the source index. This was refactored out of ReindexDataStreamIndexAction. * block-writes cannot be added after read-only Fix bug in ReindexDataStreamIndexAction. If the source index has both a block-writes and is read-only, these must be updated on the destination index. If read-only is set first, the block-writes cannot be added because settings cannot be modified.
1 parent 3d1f8d2 commit 10531df

File tree

9 files changed

+552
-66
lines changed

9 files changed

+552
-66
lines changed

‎docs/changelog/118890.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 118890
2+
summary: Add action to create index from a source index
3+
area: Data streams
4+
type: enhancement
5+
issues: []

‎x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceActionIT.java‎

Lines changed: 250 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,250 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.migrate.action;
9+
10+
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
11+
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
12+
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
13+
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest;
14+
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
15+
import org.elasticsearch.cluster.metadata.IndexMetadata;
16+
import org.elasticsearch.cluster.metadata.MappingMetadata;
17+
import org.elasticsearch.common.settings.Settings;
18+
import org.elasticsearch.common.xcontent.XContentHelper;
19+
import org.elasticsearch.common.xcontent.support.XContentMapValues;
20+
import org.elasticsearch.datastreams.DataStreamsPlugin;
21+
import org.elasticsearch.index.IndexNotFoundException;
22+
import org.elasticsearch.plugins.Plugin;
23+
import org.elasticsearch.reindex.ReindexPlugin;
24+
import org.elasticsearch.test.ESIntegTestCase;
25+
import org.elasticsearch.test.transport.MockTransportService;
26+
import org.elasticsearch.xcontent.json.JsonXContent;
27+
import org.elasticsearch.xpack.migrate.MigratePlugin;
28+
29+
import java.util.Collection;
30+
import java.util.List;
31+
import java.util.Locale;
32+
import java.util.Map;
33+
34+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
35+
import static org.elasticsearch.xpack.migrate.action.ReindexDataStreamAction.REINDEX_DATA_STREAM_FEATURE_FLAG;
36+
37+
public class CreateIndexFromSourceActionIT extends ESIntegTestCase {
38+
39+
@Override
40+
protected Collection<Class<? extends Plugin>> nodePlugins() {
41+
return List.of(MigratePlugin.class, ReindexPlugin.class, MockTransportService.TestPlugin.class, DataStreamsPlugin.class);
42+
}
43+
44+
public void testDestIndexCreated() throws Exception {
45+
assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled());
46+
47+
var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
48+
indicesAdmin().create(new CreateIndexRequest(sourceIndex)).get();
49+
50+
// create from source
51+
var destIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
52+
assertAcked(
53+
client().execute(CreateIndexFromSourceAction.INSTANCE, new CreateIndexFromSourceAction.Request(sourceIndex, destIndex))
54+
);
55+
56+
try {
57+
indicesAdmin().getIndex(new GetIndexRequest().indices(destIndex)).actionGet();
58+
} catch (IndexNotFoundException e) {
59+
fail();
60+
}
61+
}
62+
63+
public void testSettingsCopiedFromSource() throws Exception {
64+
assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled());
65+
66+
// start with a static setting
67+
var numShards = randomIntBetween(1, 10);
68+
var staticSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards).build();
69+
var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
70+
indicesAdmin().create(new CreateIndexRequest(sourceIndex, staticSettings)).get();
71+
72+
// update with a dynamic setting
73+
var numReplicas = randomIntBetween(0, 10);
74+
var dynamicSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas).build();
75+
indicesAdmin().updateSettings(new UpdateSettingsRequest(dynamicSettings, sourceIndex)).actionGet();
76+
77+
// create from source
78+
var destIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
79+
assertAcked(
80+
client().execute(CreateIndexFromSourceAction.INSTANCE, new CreateIndexFromSourceAction.Request(sourceIndex, destIndex))
81+
);
82+
83+
// assert both static and dynamic settings set on dest index
84+
var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex)).actionGet();
85+
assertEquals(numReplicas, Integer.parseInt(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_NUMBER_OF_REPLICAS)));
86+
assertEquals(numShards, Integer.parseInt(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_NUMBER_OF_SHARDS)));
87+
}
88+
89+
public void testMappingsCopiedFromSource() {
90+
assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled());
91+
92+
var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
93+
String mapping = """
94+
{
95+
"_doc":{
96+
"dynamic":"strict",
97+
"properties":{
98+
"foo1":{
99+
"type":"text"
100+
}
101+
}
102+
}
103+
}
104+
""";
105+
indicesAdmin().create(new CreateIndexRequest(sourceIndex).mapping(mapping)).actionGet();
106+
107+
// create from source
108+
var destIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
109+
assertAcked(
110+
client().execute(CreateIndexFromSourceAction.INSTANCE, new CreateIndexFromSourceAction.Request(sourceIndex, destIndex))
111+
);
112+
113+
var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(sourceIndex, destIndex)).actionGet();
114+
Map<String, MappingMetadata> mappings = mappingsResponse.mappings();
115+
var destMappings = mappings.get(destIndex).sourceAsMap();
116+
var sourceMappings = mappings.get(sourceIndex).sourceAsMap();
117+
118+
assertEquals(sourceMappings, destMappings);
119+
// sanity check specific value from dest mapping
120+
assertEquals("text", XContentMapValues.extractValue("properties.foo1.type", destMappings));
121+
}
122+
123+
public void testSettingsOverridden() throws Exception {
124+
assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled());
125+
126+
var numShardsSource = randomIntBetween(1, 10);
127+
var numReplicasSource = randomIntBetween(0, 10);
128+
var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
129+
var sourceSettings = Settings.builder()
130+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShardsSource)
131+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicasSource)
132+
.build();
133+
indicesAdmin().create(new CreateIndexRequest(sourceIndex, sourceSettings)).get();
134+
135+
boolean overrideNumShards = randomBoolean();
136+
Settings settingsOverride = overrideNumShards
137+
? Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShardsSource + 1).build()
138+
: Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicasSource + 1).build();
139+
140+
// create from source
141+
var destIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
142+
assertAcked(
143+
client().execute(
144+
CreateIndexFromSourceAction.INSTANCE,
145+
new CreateIndexFromSourceAction.Request(sourceIndex, destIndex, settingsOverride, Map.of())
146+
)
147+
);
148+
149+
// assert settings overridden
150+
int expectedShards = overrideNumShards ? numShardsSource + 1 : numShardsSource;
151+
int expectedReplicas = overrideNumShards ? numReplicasSource : numReplicasSource + 1;
152+
var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex)).actionGet();
153+
assertEquals(expectedShards, Integer.parseInt(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_NUMBER_OF_SHARDS)));
154+
assertEquals(expectedReplicas, Integer.parseInt(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_NUMBER_OF_REPLICAS)));
155+
}
156+
157+
public void testSettingsNullOverride() throws Exception {
158+
assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled());
159+
160+
var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
161+
var sourceSettings = Settings.builder().put(IndexMetadata.SETTING_BLOCKS_WRITE, true).build();
162+
indicesAdmin().create(new CreateIndexRequest(sourceIndex, sourceSettings)).get();
163+
164+
Settings settingsOverride = Settings.builder().putNull(IndexMetadata.SETTING_BLOCKS_WRITE).build();
165+
166+
// create from source
167+
var destIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
168+
assertAcked(
169+
client().execute(
170+
CreateIndexFromSourceAction.INSTANCE,
171+
new CreateIndexFromSourceAction.Request(sourceIndex, destIndex, settingsOverride, Map.of())
172+
)
173+
);
174+
175+
// assert settings overridden
176+
var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex)).actionGet();
177+
assertNull(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_BLOCKS_WRITE));
178+
}
179+
180+
public void testMappingsOverridden() {
181+
assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled());
182+
183+
var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
184+
String sourceMapping = """
185+
{
186+
"_doc":{
187+
"dynamic":"strict",
188+
"properties":{
189+
"foo1":{
190+
"type":"text"
191+
},
192+
"foo2":{
193+
"type":"boolean"
194+
}
195+
}
196+
}
197+
}
198+
""";
199+
indicesAdmin().create(new CreateIndexRequest(sourceIndex).mapping(sourceMapping)).actionGet();
200+
201+
String mappingOverrideStr = """
202+
{
203+
"_doc":{
204+
"dynamic":"strict",
205+
"properties":{
206+
"foo1":{
207+
"type":"integer"
208+
},
209+
"foo3": {
210+
"type":"keyword"
211+
}
212+
}
213+
}
214+
}
215+
""";
216+
var mappingOverride = XContentHelper.convertToMap(JsonXContent.jsonXContent, mappingOverrideStr, false);
217+
218+
// create from source
219+
var destIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
220+
assertAcked(
221+
client().execute(
222+
CreateIndexFromSourceAction.INSTANCE,
223+
new CreateIndexFromSourceAction.Request(sourceIndex, destIndex, Settings.EMPTY, mappingOverride)
224+
)
225+
);
226+
227+
var mappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(destIndex)).actionGet();
228+
Map<String, MappingMetadata> mappings = mappingsResponse.mappings();
229+
var destMappings = mappings.get(destIndex).sourceAsMap();
230+
231+
String expectedMappingStr = """
232+
{
233+
"dynamic":"strict",
234+
"properties":{
235+
"foo1":{
236+
"type":"integer"
237+
},
238+
"foo2": {
239+
"type":"boolean"
240+
},
241+
"foo3": {
242+
"type":"keyword"
243+
}
244+
}
245+
}
246+
""";
247+
var expectedMapping = XContentHelper.convertToMap(JsonXContent.jsonXContent, expectedMappingStr, false);
248+
assertEquals(expectedMapping, destMappings);
249+
}
250+
}

‎x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexIT.java‎ renamed to ‎x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/ReindexDatastreamIndexTransportActionIT.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@
5353
import static org.elasticsearch.xpack.migrate.action.ReindexDataStreamAction.REINDEX_DATA_STREAM_FEATURE_FLAG;
5454
import static org.hamcrest.Matchers.equalTo;
5555

56-
public class ReindexDatastreamIndexIT extends ESIntegTestCase {
56+
public class ReindexDatastreamIndexTransportActionIT extends ESIntegTestCase {
5757

5858
private static final String MAPPING = """
5959
{
@@ -126,12 +126,14 @@ public void testDestIndexContainsDocs() throws Exception {
126126
assertHitCount(prepareSearch(response.getDestIndex()).setSize(0), numDocs);
127127
}
128128

129-
public void testSetSourceToReadOnly() throws Exception {
129+
public void testSetSourceToBlockWrites() throws Exception {
130130
assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled());
131131

132+
var settings = randomBoolean() ? Settings.builder().put(IndexMetadata.SETTING_BLOCKS_WRITE, true).build() : Settings.EMPTY;
133+
132134
// empty source index
133135
var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
134-
indicesAdmin().create(new CreateIndexRequest(sourceIndex)).get();
136+
indicesAdmin().create(new CreateIndexRequest(sourceIndex, settings)).get();
135137

136138
// call reindex
137139
client().execute(ReindexDataStreamIndexAction.INSTANCE, new ReindexDataStreamIndexAction.Request(sourceIndex)).actionGet();

‎x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/MigratePlugin.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@
3434
import org.elasticsearch.xcontent.ParseField;
3535
import org.elasticsearch.xpack.migrate.action.CancelReindexDataStreamAction;
3636
import org.elasticsearch.xpack.migrate.action.CancelReindexDataStreamTransportAction;
37+
import org.elasticsearch.xpack.migrate.action.CreateIndexFromSourceAction;
38+
import org.elasticsearch.xpack.migrate.action.CreateIndexFromSourceTransportAction;
3739
import org.elasticsearch.xpack.migrate.action.GetMigrationReindexStatusAction;
3840
import org.elasticsearch.xpack.migrate.action.GetMigrationReindexStatusTransportAction;
3941
import org.elasticsearch.xpack.migrate.action.ReindexDataStreamAction;
@@ -87,6 +89,7 @@ public List<RestHandler> getRestHandlers(
8789
actions.add(new ActionHandler<>(GetMigrationReindexStatusAction.INSTANCE, GetMigrationReindexStatusTransportAction.class));
8890
actions.add(new ActionHandler<>(CancelReindexDataStreamAction.INSTANCE, CancelReindexDataStreamTransportAction.class));
8991
actions.add(new ActionHandler<>(ReindexDataStreamIndexAction.INSTANCE, ReindexDataStreamIndexTransportAction.class));
92+
actions.add(new ActionHandler<>(CreateIndexFromSourceAction.INSTANCE, CreateIndexFromSourceTransportAction.class));
9093
}
9194
return actions;
9295
}

0 commit comments

Comments
 (0)