Skip to content

Commit 68251d4

Browse files
gwbrowngwbrown
authored
[7.17] Register Feature migration persistent task state named XContent (#84192) (#84258)
In addition to the backport, this commit also adds SystemIndexMigrationTaskParams/State to writeableRegistry of transport client This will backport the following commits from master to 7.17: Register Feature migration persistent task state named XContent (#84192)
1 parent d90d673 commit 68251d4

14 files changed

+553
-271
lines changed

‎docs/changelog/84192.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 84192
2+
summary: Registration of `SystemIndexMigrationTask` named xcontent objects
3+
area: Infra/Core
4+
type: bug
5+
issues:
6+
- 84115

‎modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java‎

Lines changed: 283 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,283 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
9+
package org.elasticsearch.migration;
10+
11+
import org.elasticsearch.Version;
12+
import org.elasticsearch.action.ActionListener;
13+
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
14+
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
15+
import org.elasticsearch.action.admin.indices.stats.IndexStats;
16+
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
17+
import org.elasticsearch.action.index.IndexRequestBuilder;
18+
import org.elasticsearch.action.support.ActiveShardCount;
19+
import org.elasticsearch.client.Client;
20+
import org.elasticsearch.cluster.ClusterState;
21+
import org.elasticsearch.cluster.metadata.IndexMetadata;
22+
import org.elasticsearch.cluster.metadata.Metadata;
23+
import org.elasticsearch.cluster.service.ClusterService;
24+
import org.elasticsearch.common.Strings;
25+
import org.elasticsearch.common.settings.Settings;
26+
import org.elasticsearch.indices.AssociatedIndexDescriptor;
27+
import org.elasticsearch.indices.SystemIndexDescriptor;
28+
import org.elasticsearch.plugins.Plugin;
29+
import org.elasticsearch.plugins.SystemIndexPlugin;
30+
import org.elasticsearch.test.ESIntegTestCase;
31+
import org.elasticsearch.xcontent.XContentBuilder;
32+
import org.elasticsearch.xcontent.XContentType;
33+
import org.elasticsearch.xcontent.json.JsonXContent;
34+
import org.junit.Assert;
35+
import org.junit.Before;
36+
37+
import java.io.IOException;
38+
import java.util.ArrayList;
39+
import java.util.Arrays;
40+
import java.util.Collection;
41+
import java.util.Collections;
42+
import java.util.List;
43+
import java.util.Map;
44+
import java.util.Optional;
45+
import java.util.Set;
46+
import java.util.concurrent.atomic.AtomicReference;
47+
import java.util.function.BiConsumer;
48+
import java.util.function.Function;
49+
50+
import static org.hamcrest.Matchers.containsInAnyOrder;
51+
import static org.hamcrest.Matchers.equalTo;
52+
import static org.hamcrest.Matchers.is;
53+
54+
public abstract class AbstractFeatureMigrationIntegTest extends ESIntegTestCase {
55+
56+
static final String VERSION_META_KEY = "version";
57+
static final Version META_VERSION = Version.CURRENT;
58+
static final String DESCRIPTOR_MANAGED_META_KEY = "desciptor_managed";
59+
static final String DESCRIPTOR_INTERNAL_META_KEY = "descriptor_internal";
60+
static final String FEATURE_NAME = "A-test-feature"; // Sorts alphabetically before the feature from MultiFeatureMigrationIT
61+
static final String ORIGIN = AbstractFeatureMigrationIntegTest.class.getSimpleName();
62+
static final String FlAG_SETTING_KEY = IndexMetadata.INDEX_PRIORITY_SETTING.getKey();
63+
static final String INTERNAL_MANAGED_INDEX_NAME = ".int-man-old";
64+
static final int INDEX_DOC_COUNT = 100; // arbitrarily chosen
65+
static final int INTERNAL_MANAGED_FLAG_VALUE = 1;
66+
public static final Version NEEDS_UPGRADE_VERSION = Version.V_6_0_0;
67+
68+
static final SystemIndexDescriptor EXTERNAL_UNMANAGED = SystemIndexDescriptor.builder()
69+
.setIndexPattern(".ext-unman-*")
70+
.setType(SystemIndexDescriptor.Type.EXTERNAL_UNMANAGED)
71+
.setOrigin(ORIGIN)
72+
.setVersionMetaKey(VERSION_META_KEY)
73+
.setAllowedElasticProductOrigins(Collections.singletonList(ORIGIN))
74+
.setMinimumNodeVersion(NEEDS_UPGRADE_VERSION)
75+
.setPriorSystemIndexDescriptors(Collections.emptyList())
76+
77+
.build();
78+
static final SystemIndexDescriptor INTERNAL_UNMANAGED = SystemIndexDescriptor.builder()
79+
.setIndexPattern(".int-unman-*")
80+
.setType(SystemIndexDescriptor.Type.INTERNAL_UNMANAGED)
81+
.setOrigin(ORIGIN)
82+
.setVersionMetaKey(VERSION_META_KEY)
83+
.setAllowedElasticProductOrigins(Collections.emptyList())
84+
.setMinimumNodeVersion(NEEDS_UPGRADE_VERSION)
85+
.setPriorSystemIndexDescriptors(Collections.emptyList())
86+
.build();
87+
88+
static final SystemIndexDescriptor INTERNAL_MANAGED = SystemIndexDescriptor.builder()
89+
.setIndexPattern(".int-man-*")
90+
.setAliasName(".internal-managed-alias")
91+
.setPrimaryIndex(INTERNAL_MANAGED_INDEX_NAME)
92+
.setType(SystemIndexDescriptor.Type.INTERNAL_MANAGED)
93+
.setSettings(createSimpleSettings(NEEDS_UPGRADE_VERSION, INTERNAL_MANAGED_FLAG_VALUE))
94+
.setMappings(createSimpleMapping(true, true, false))
95+
.setOrigin(ORIGIN)
96+
.setVersionMetaKey(VERSION_META_KEY)
97+
.setAllowedElasticProductOrigins(Collections.emptyList())
98+
.setMinimumNodeVersion(NEEDS_UPGRADE_VERSION)
99+
.setPriorSystemIndexDescriptors(Collections.emptyList())
100+
.setIndexType("doc") // This simulates `.tasks`, which uses a nonstandard type name. EXTERNAL_MANAGED tests the default type.
101+
.build();
102+
static final int INTERNAL_UNMANAGED_FLAG_VALUE = 2;
103+
static final int EXTERNAL_MANAGED_FLAG_VALUE = 3;
104+
static final SystemIndexDescriptor EXTERNAL_MANAGED = SystemIndexDescriptor.builder()
105+
.setIndexPattern(".ext-man-*")
106+
.setAliasName(".external-managed-alias")
107+
.setPrimaryIndex(".ext-man-old")
108+
.setType(SystemIndexDescriptor.Type.EXTERNAL_MANAGED)
109+
.setSettings(createSimpleSettings(NEEDS_UPGRADE_VERSION, EXTERNAL_MANAGED_FLAG_VALUE))
110+
.setMappings(createSimpleMapping(true, false, true))
111+
.setOrigin(ORIGIN)
112+
.setVersionMetaKey(VERSION_META_KEY)
113+
.setAllowedElasticProductOrigins(Collections.singletonList(ORIGIN))
114+
.setMinimumNodeVersion(NEEDS_UPGRADE_VERSION)
115+
.setPriorSystemIndexDescriptors(Collections.emptyList())
116+
.build();
117+
static final int EXTERNAL_UNMANAGED_FLAG_VALUE = 4;
118+
static final String ASSOCIATED_INDEX_NAME = ".my-associated-idx";
119+
120+
@Before
121+
public void setupTestPlugin() {
122+
TestPlugin.preMigrationHook.set((state) -> Collections.emptyMap());
123+
TestPlugin.postMigrationHook.set((state, metadata) -> {});
124+
}
125+
126+
public void createSystemIndexForDescriptor(SystemIndexDescriptor descriptor) throws InterruptedException {
127+
Assert.assertTrue(
128+
"the strategy used below to create index names for descriptors without a primary index name only works for simple patterns",
129+
descriptor.getIndexPattern().endsWith("*")
130+
);
131+
String indexName = Optional.ofNullable(descriptor.getPrimaryIndex()).orElse(descriptor.getIndexPattern().replace("*", "old"));
132+
CreateIndexRequestBuilder createRequest = prepareCreate(indexName);
133+
createRequest.setWaitForActiveShards(ActiveShardCount.ALL);
134+
if (descriptor.getSettings() != null) {
135+
createRequest.setSettings(
136+
Settings.builder()
137+
.put("index.version.created", Version.CURRENT)
138+
.put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
139+
.build()
140+
);
141+
} else {
142+
createRequest.setSettings(
143+
createSimpleSettings(
144+
NEEDS_UPGRADE_VERSION,
145+
descriptor.isInternal() ? INTERNAL_UNMANAGED_FLAG_VALUE : EXTERNAL_UNMANAGED_FLAG_VALUE
146+
)
147+
);
148+
}
149+
if (descriptor.getMappings() == null) {
150+
createRequest.addMapping("doc", createSimpleMapping(false, descriptor.isInternal(), false), XContentType.JSON);
151+
}
152+
CreateIndexResponse response = createRequest.get();
153+
Assert.assertTrue(response.isShardsAcknowledged());
154+
155+
List<IndexRequestBuilder> docs = new ArrayList<>(INDEX_DOC_COUNT);
156+
for (int i = 0; i < INDEX_DOC_COUNT; i++) {
157+
docs.add(
158+
ESIntegTestCase.client().prepareIndex(indexName, "_doc").setId(Integer.toString(i)).setSource("some_field", "words words")
159+
);
160+
}
161+
indexRandom(true, docs);
162+
IndicesStatsResponse indexStats = ESIntegTestCase.client().admin().indices().prepareStats(indexName).setDocs(true).get();
163+
Assert.assertThat(indexStats.getIndex(indexName).getTotal().getDocs().getCount(), is((long) INDEX_DOC_COUNT));
164+
}
165+
166+
static Settings createSimpleSettings(Version creationVersion, int flagSettingValue) {
167+
return Settings.builder()
168+
.put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
169+
.put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
170+
.put(FlAG_SETTING_KEY, flagSettingValue)
171+
.put("index.version.created", creationVersion)
172+
.build();
173+
}
174+
175+
static String createSimpleMapping(boolean descriptorManaged, boolean descriptorInternal, boolean useStandardType) {
176+
try (XContentBuilder builder = JsonXContent.contentBuilder()) {
177+
builder.startObject();
178+
if (useStandardType) {
179+
builder.startObject("_doc");
180+
} else {
181+
builder.startObject("doc");
182+
}
183+
{
184+
builder.startObject("_meta");
185+
builder.field(VERSION_META_KEY, META_VERSION);
186+
builder.field(DESCRIPTOR_MANAGED_META_KEY, descriptorManaged);
187+
builder.field(DESCRIPTOR_INTERNAL_META_KEY, descriptorInternal);
188+
builder.endObject();
189+
190+
builder.field("dynamic", "strict");
191+
builder.startObject("properties");
192+
{
193+
builder.startObject("some_field");
194+
builder.field("type", "keyword");
195+
builder.endObject();
196+
}
197+
builder.endObject();
198+
}
199+
builder.endObject();
200+
builder.endObject();
201+
return Strings.toString(builder);
202+
} catch (IOException e) {
203+
// Just rethrow, it should be impossible for this to throw here
204+
throw new AssertionError(e);
205+
}
206+
}
207+
208+
public void assertIndexHasCorrectProperties(
209+
Metadata metadata,
210+
String indexName,
211+
int settingsFlagValue,
212+
boolean isManaged,
213+
boolean isInternal,
214+
Collection<String> aliasNames
215+
) {
216+
IndexMetadata imd = metadata.index(indexName);
217+
assertThat(imd.getSettings().get(FlAG_SETTING_KEY), equalTo(Integer.toString(settingsFlagValue)));
218+
final Map<String, Object> mapping = imd.mapping().getSourceAsMap();
219+
@SuppressWarnings("unchecked")
220+
final Map<String, Object> meta = (Map<String, Object>) mapping.get("_meta");
221+
assertThat(meta.get(DESCRIPTOR_MANAGED_META_KEY), is(isManaged));
222+
assertThat(meta.get(DESCRIPTOR_INTERNAL_META_KEY), is(isInternal));
223+
224+
assertThat(imd.isSystem(), is(true));
225+
226+
Set<String> actualAliasNames = imd.getAliases().keySet();
227+
assertThat(actualAliasNames, containsInAnyOrder(aliasNames.toArray()));
228+
229+
IndicesStatsResponse indexStats = client().admin().indices().prepareStats(imd.getIndex().getName()).setDocs(true).get();
230+
assertNotNull(indexStats);
231+
final IndexStats thisIndexStats = indexStats.getIndex(imd.getIndex().getName());
232+
assertNotNull(thisIndexStats);
233+
assertNotNull(thisIndexStats.getTotal());
234+
assertNotNull(thisIndexStats.getTotal().getDocs());
235+
assertThat(thisIndexStats.getTotal().getDocs().getCount(), is((long) INDEX_DOC_COUNT));
236+
}
237+
238+
public static class TestPlugin extends Plugin implements SystemIndexPlugin {
239+
public static final AtomicReference<Function<ClusterState, Map<String, Object>>> preMigrationHook = new AtomicReference<>();
240+
public static final AtomicReference<BiConsumer<ClusterState, Map<String, Object>>> postMigrationHook = new AtomicReference<>();
241+
242+
public TestPlugin() {
243+
244+
}
245+
246+
@Override
247+
public String getFeatureName() {
248+
return FEATURE_NAME;
249+
}
250+
251+
@Override
252+
public String getFeatureDescription() {
253+
return "a plugin for testing system index migration";
254+
}
255+
256+
@Override
257+
public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
258+
return Arrays.asList(INTERNAL_MANAGED, INTERNAL_UNMANAGED, EXTERNAL_MANAGED, EXTERNAL_UNMANAGED);
259+
}
260+
261+
@Override
262+
public Collection<AssociatedIndexDescriptor> getAssociatedIndexDescriptors() {
263+
264+
return Collections.singletonList(new AssociatedIndexDescriptor(ASSOCIATED_INDEX_NAME, TestPlugin.class.getCanonicalName()));
265+
}
266+
267+
@Override
268+
public void prepareForIndicesMigration(ClusterService clusterService, Client client, ActionListener<Map<String, Object>> listener) {
269+
listener.onResponse(preMigrationHook.get().apply(clusterService.state()));
270+
}
271+
272+
@Override
273+
public void indicesMigrationComplete(
274+
Map<String, Object> preUpgradeMetadata,
275+
ClusterService clusterService,
276+
Client client,
277+
ActionListener<Boolean> listener
278+
) {
279+
postMigrationHook.get().accept(clusterService.state(), preUpgradeMetadata);
280+
listener.onResponse(true);
281+
}
282+
}
283+
}

0 commit comments

Comments
 (0)