Skip to content

Commit c0f5e00

Browse files
authored
[Transform] Check alias during update (elastic#124825)
When the Transform System Index has been reindexed and aliased, we should check the Transform Update index against the alias when updating the Transform Config.
1 parent be396ae commit c0f5e00

File tree

6 files changed

+98
-6
lines changed

6 files changed

+98
-6
lines changed

docs/changelog/124825.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 124825
2+
summary: Check alias during update
3+
area: Transform
4+
type: bug
5+
issues: []

x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformOldTransformsIT.java

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
import org.elasticsearch.common.settings.Settings;
1717
import org.elasticsearch.core.Strings;
1818
import org.elasticsearch.core.TimeValue;
19+
import org.elasticsearch.index.reindex.ReindexAction;
20+
import org.elasticsearch.index.reindex.ReindexRequest;
1921
import org.elasticsearch.xcontent.XContentBuilder;
2022
import org.elasticsearch.xcontent.XContentFactory;
2123
import org.elasticsearch.xcontent.XContentType;
@@ -25,12 +27,16 @@
2527
import org.elasticsearch.xpack.core.transform.TransformDeprecations;
2628
import org.elasticsearch.xpack.core.transform.TransformField;
2729
import org.elasticsearch.xpack.core.transform.action.GetTransformAction;
30+
import org.elasticsearch.xpack.core.transform.action.PutTransformAction;
2831
import org.elasticsearch.xpack.core.transform.action.StartTransformAction;
2932
import org.elasticsearch.xpack.core.transform.action.StopTransformAction;
3033
import org.elasticsearch.xpack.core.transform.action.UpdateTransformAction;
34+
import org.elasticsearch.xpack.core.transform.transforms.DestConfig;
35+
import org.elasticsearch.xpack.core.transform.transforms.SourceConfig;
3136
import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
3237
import org.elasticsearch.xpack.core.transform.transforms.TransformConfigUpdate;
3338
import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants;
39+
import org.elasticsearch.xpack.core.transform.transforms.pivot.PivotConfigTests;
3440
import org.elasticsearch.xpack.core.transform.utils.TransformConfigVersionUtils;
3541
import org.elasticsearch.xpack.transform.TransformSingleNodeTestCase;
3642
import org.elasticsearch.xpack.transform.persistence.TransformInternalIndex;
@@ -253,4 +259,65 @@ public void testStartReplacesDeprecatedTransformSettings() throws Exception {
253259
assertMaxPageSearchSizeInSettings(transformId, expectedMaxPageSearchSize);
254260
}
255261

262+
public void testMigratedTransformIndex() {
263+
// create transform
264+
var sourceIndex = "source-index";
265+
createSourceIndex(sourceIndex);
266+
var transformId = "transform-migrated-system-index";
267+
268+
var sourceConfig = new SourceConfig(sourceIndex);
269+
var destConfig = new DestConfig("some-dest-index", null, null);
270+
var config = new TransformConfig(
271+
transformId,
272+
sourceConfig,
273+
destConfig,
274+
null,
275+
null,
276+
null,
277+
PivotConfigTests.randomPivotConfig(),
278+
null,
279+
null,
280+
null,
281+
null,
282+
null,
283+
null,
284+
null
285+
);
286+
var putTransform = new PutTransformAction.Request(config, true, TimeValue.THIRTY_SECONDS);
287+
assertTrue(client().execute(PutTransformAction.INSTANCE, putTransform).actionGet().isAcknowledged());
288+
289+
// simulate migration by reindexing and aliasing
290+
var newSystemIndex = TransformInternalIndexConstants.LATEST_INDEX_NAME + "-reindexed";
291+
var reindexRequest = new ReindexRequest();
292+
reindexRequest.setSourceIndices(TransformInternalIndexConstants.LATEST_INDEX_NAME);
293+
reindexRequest.setDestIndex(newSystemIndex);
294+
reindexRequest.setRefresh(true);
295+
client().execute(ReindexAction.INSTANCE, reindexRequest).actionGet();
296+
297+
var aliasesRequest = admin().indices().prepareAliases(TimeValue.THIRTY_SECONDS, TimeValue.THIRTY_SECONDS);
298+
aliasesRequest.removeIndex(TransformInternalIndexConstants.LATEST_INDEX_NAME);
299+
aliasesRequest.addAlias(newSystemIndex, TransformInternalIndexConstants.LATEST_INDEX_NAME);
300+
aliasesRequest.execute().actionGet();
301+
302+
// update should succeed
303+
var updateConfig = new TransformConfigUpdate(
304+
sourceConfig,
305+
new DestConfig("some-new-dest-index", null, null),
306+
null,
307+
null,
308+
null,
309+
null,
310+
null,
311+
null
312+
);
313+
var updateRequest = new UpdateTransformAction.Request(updateConfig, transformId, true, TimeValue.THIRTY_SECONDS);
314+
client().execute(UpdateTransformAction.INSTANCE, updateRequest).actionGet();
315+
316+
// verify update succeeded
317+
var getTransformRequest = new GetTransformAction.Request(transformId);
318+
var getTransformResponse = client().execute(GetTransformAction.INSTANCE, getTransformRequest).actionGet();
319+
var transformConfig = getTransformResponse.getTransformConfigurations().get(0);
320+
assertThat(transformConfig.getDestination().getIndex(), equalTo("some-new-dest-index"));
321+
}
322+
256323
}

x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/persistence/TransformConfigManagerTests.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,8 @@ public class TransformConfigManagerTests extends TransformSingleNodeTestCase {
7878

7979
@Before
8080
public void createComponents() {
81-
clusterService = mock(ClusterService.class);
81+
clusterService = mock();
82+
when(clusterService.state()).thenReturn(ClusterState.EMPTY_STATE);
8283
transformConfigManager = new IndexBasedTransformConfigManager(
8384
clusterService,
8485
TestIndexNameExpressionResolver.newInstance(),

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransformUpdater.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -252,7 +252,7 @@ private static void updateTransformStateAndGetLastCheckpoint(
252252
long lastCheckpoint = currentState.v1().getTransformState().getCheckpoint();
253253

254254
// if: the state is stored on the latest index, it does not need an update
255-
if (currentState.v2().getIndex().equals(TransformInternalIndexConstants.LATEST_INDEX_VERSIONED_NAME)) {
255+
if (transformConfigManager.isLatestTransformIndex(currentState.v2().getIndex())) {
256256
listener.onResponse(lastCheckpoint);
257257
return;
258258
}
@@ -283,8 +283,7 @@ private static void updateTransformCheckpoint(
283283
ActionListener<Boolean> listener
284284
) {
285285
transformConfigManager.getTransformCheckpointForUpdate(transformId, lastCheckpoint, ActionListener.wrap(checkpointAndVersion -> {
286-
if (checkpointAndVersion == null
287-
|| checkpointAndVersion.v2().getIndex().equals(TransformInternalIndexConstants.LATEST_INDEX_VERSIONED_NAME)) {
286+
if (checkpointAndVersion == null || transformConfigManager.isLatestTransformIndex(checkpointAndVersion.v2().getIndex())) {
288287
listener.onResponse(true);
289288
return;
290289
}

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@ public void updateTransformConfiguration(
170170
listener.onFailure(conflictStatusException("Cannot update Transform while the Transform feature is upgrading."));
171171
return;
172172
}
173-
if (seqNoPrimaryTermAndIndex.getIndex().equals(TransformInternalIndexConstants.LATEST_INDEX_NAME)) {
173+
if (isLatestTransformIndex(seqNoPrimaryTermAndIndex.getIndex())) {
174174
// update the config in the same, current index using optimistic concurrency control
175175
putTransformConfiguration(transformConfig, DocWriteRequest.OpType.INDEX, seqNoPrimaryTermAndIndex, listener);
176176
} else {
@@ -180,6 +180,21 @@ public void updateTransformConfiguration(
180180
}
181181
}
182182

183+
@Override
184+
public boolean isLatestTransformIndex(String indexName) {
185+
if (TransformInternalIndexConstants.LATEST_INDEX_NAME.equals(indexName)) {
186+
return true;
187+
}
188+
189+
// in some cases, the System Index gets reindexed and LATEST_INDEX_NAME is now an alias pointing to that reindexed index
190+
// this mostly likely happens after the SystemIndexMigrator ran
191+
// we need to check if the LATEST_INDEX_NAME is now an alias and points to the indexName
192+
var metadata = clusterService.state().projectState().metadata();
193+
var indicesForAlias = metadata.aliasedIndices(TransformInternalIndexConstants.LATEST_INDEX_NAME);
194+
var index = metadata.index(indexName);
195+
return index != null && indicesForAlias.contains(index.getIndex());
196+
}
197+
183198
@Override
184199
public void deleteOldTransformConfigurations(String transformId, ActionListener<Boolean> listener) {
185200
if (isUpgrading()) {
@@ -697,7 +712,7 @@ public void putOrUpdateTransformStoredDoc(
697712
// could have been called, see gh#80073
698713
indexRequest.opType(DocWriteRequest.OpType.INDEX);
699714
// if on the latest index use optimistic concurrency control in addition
700-
if (seqNoPrimaryTermAndIndex.getIndex().equals(TransformInternalIndexConstants.LATEST_INDEX_NAME)) {
715+
if (isLatestTransformIndex(seqNoPrimaryTermAndIndex.getIndex())) {
701716
indexRequest.setIfSeqNo(seqNoPrimaryTermAndIndex.getSeqNo())
702717
.setIfPrimaryTerm(seqNoPrimaryTermAndIndex.getPrimaryTerm());
703718
}

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformConfigManager.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint;
1616
import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
1717
import org.elasticsearch.xpack.core.transform.transforms.TransformStoredDoc;
18+
import org.elasticsearch.xpack.core.transform.transforms.persistence.TransformInternalIndexConstants;
1819

1920
import java.util.Collection;
2021
import java.util.Collections;
@@ -206,4 +207,8 @@ void getTransformStoredDoc(
206207
void getTransformStoredDocs(Collection<String> transformIds, TimeValue timeout, ActionListener<List<TransformStoredDoc>> listener);
207208

208209
void refresh(ActionListener<Boolean> listener);
210+
211+
default boolean isLatestTransformIndex(String indexName) {
212+
return TransformInternalIndexConstants.LATEST_INDEX_NAME.equals(indexName);
213+
}
209214
}

0 commit comments

Comments
 (0)