Skip to content

Commit 40b1e3e

Browse files
prwhelanprwhelan
authored
[Transform] Auto-migrate max_page_search_size (elastic#119348) (elastic#119697)
`max_page_search_size` had been deprecated in Pivot and migrated to Settings in ES 7.x. We are removing the key in Pivot in 9.x. It is still possible to create Pivots with `max_page_search_size`, though a deprecation is logged. To prep for its removal, this change automatically migrates `max_page_search_size` from Pivot to Settings as if the user called `_update`. - `PUT _transform` with `max_page_search_size` in Pivot will migrate the value to Settings as part of the Transform's creation. The user will receive an additional deprecation warning informing of this migration. - Existing transforms with `max_page_search_size` in Pivot will migrate the value to Settings as part of node assignment during start. Transform Messages and Elasticsearch logs will record this event so users will know what happened.
1 parent 12f4357 commit 40b1e3e

File tree

14 files changed

+580
-120
lines changed

14 files changed

+580
-120
lines changed

‎docs/changelog/119348.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 119348
2+
summary: Auto-migrate `max_page_search_size`
3+
area: Transform
4+
type: enhancement
5+
issues: []

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMessages.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,9 @@ public class TransformMessages {
8989
public static final String INVALID_ID = "Invalid {0}; ''{1}'' can contain lowercase alphanumeric (a-z and 0-9), hyphens or "
9090
+ "underscores; must start and end with alphanumeric";
9191

92+
public static final String MAX_PAGE_SEARCH_SIZE_MIGRATION =
93+
"Automatically migrating [max_page_search_size] from the pivot to the transform setting.";
94+
9295
private TransformMessages() {}
9396

9497
/**

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java‎

Lines changed: 33 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -615,24 +615,15 @@ public static TransformConfig rewriteForUpdate(final TransformConfig transformCo
615615

616616
private static TransformConfig applyRewriteForUpdate(Builder builder) {
617617
// 1. Move pivot.max_page_size_search to settings.max_page_size_search
618-
if (builder.getPivotConfig() != null && builder.getPivotConfig().getMaxPageSearchSize() != null) {
619-
620-
// find maxPageSearchSize value
621-
Integer maxPageSearchSizeDeprecated = builder.getPivotConfig().getMaxPageSearchSize();
622-
Integer maxPageSearchSize = builder.getSettings().getMaxPageSearchSize() != null
623-
? builder.getSettings().getMaxPageSearchSize()
624-
: maxPageSearchSizeDeprecated;
618+
migrateMaxPageSearchSize(builder);
625619

626-
// create a new pivot config but set maxPageSearchSize to null
627-
builder.setPivotConfig(
628-
new PivotConfig(builder.getPivotConfig().getGroupConfig(), builder.getPivotConfig().getAggregationConfig(), null)
629-
);
630-
// create new settings with maxPageSearchSize
620+
// 2. set dates_as_epoch_millis to true for transforms < 7.11 to keep BWC
621+
if (builder.getVersion() != null && builder.getVersion().before(TransformConfigVersion.V_7_11_0)) {
631622
builder.setSettings(
632623
new SettingsConfig(
633-
maxPageSearchSize,
624+
builder.getSettings().getMaxPageSearchSize(),
634625
builder.getSettings().getDocsPerSecond(),
635-
builder.getSettings().getDatesAsEpochMillis(),
626+
true,
636627
builder.getSettings().getAlignCheckpoints(),
637628
builder.getSettings().getUsePit(),
638629
builder.getSettings().getDeduceMappings(),
@@ -642,14 +633,14 @@ private static TransformConfig applyRewriteForUpdate(Builder builder) {
642633
);
643634
}
644635

645-
// 2. set dates_as_epoch_millis to true for transforms < 7.11 to keep BWC
646-
if (builder.getVersion() != null && builder.getVersion().before(TransformConfigVersion.V_7_11_0)) {
636+
// 3. set align_checkpoints to false for transforms < 7.15 to keep BWC
637+
if (builder.getVersion() != null && builder.getVersion().before(TransformConfigVersion.V_7_15_0)) {
647638
builder.setSettings(
648639
new SettingsConfig(
649640
builder.getSettings().getMaxPageSearchSize(),
650641
builder.getSettings().getDocsPerSecond(),
651-
true,
652-
builder.getSettings().getAlignCheckpoints(),
642+
builder.getSettings().getDatesAsEpochMillis(),
643+
false,
653644
builder.getSettings().getUsePit(),
654645
builder.getSettings().getDeduceMappings(),
655646
builder.getSettings().getNumFailureRetries(),
@@ -658,23 +649,41 @@ private static TransformConfig applyRewriteForUpdate(Builder builder) {
658649
);
659650
}
660651

661-
// 3. set align_checkpoints to false for transforms < 7.15 to keep BWC
662-
if (builder.getVersion() != null && builder.getVersion().before(TransformConfigVersion.V_7_15_0)) {
652+
return builder.setVersion(TransformConfigVersion.CURRENT).build();
653+
}
654+
655+
public boolean shouldAutoMigrateMaxPageSearchSize() {
656+
return getPivotConfig() != null && getPivotConfig().getMaxPageSearchSize() != null;
657+
}
658+
659+
public static Builder migrateMaxPageSearchSize(Builder builder) {
660+
if (builder.getPivotConfig() != null && builder.getPivotConfig().getMaxPageSearchSize() != null) {
661+
662+
// find maxPageSearchSize value
663+
Integer maxPageSearchSizeDeprecated = builder.getPivotConfig().getMaxPageSearchSize();
664+
Integer maxPageSearchSize = builder.getSettings().getMaxPageSearchSize() != null
665+
? builder.getSettings().getMaxPageSearchSize()
666+
: maxPageSearchSizeDeprecated;
667+
668+
// create a new pivot config but set maxPageSearchSize to null
669+
builder.setPivotConfig(
670+
new PivotConfig(builder.getPivotConfig().getGroupConfig(), builder.getPivotConfig().getAggregationConfig(), null)
671+
);
672+
// create new settings with maxPageSearchSize
663673
builder.setSettings(
664674
new SettingsConfig(
665-
builder.getSettings().getMaxPageSearchSize(),
675+
maxPageSearchSize,
666676
builder.getSettings().getDocsPerSecond(),
667677
builder.getSettings().getDatesAsEpochMillis(),
668-
false,
678+
builder.getSettings().getAlignCheckpoints(),
669679
builder.getSettings().getUsePit(),
670680
builder.getSettings().getDeduceMappings(),
671681
builder.getSettings().getNumFailureRetries(),
672682
builder.getSettings().getUnattended()
673683
)
674684
);
675685
}
676-
677-
return builder.setVersion(TransformConfigVersion.CURRENT).build();
686+
return builder;
678687
}
679688

680689
public static Builder builder() {

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,14 @@ public static TransformConfig randomTransformConfigWithSettings(SettingsConfig s
152152
latestConfig = LatestConfigTests.randomLatestConfig();
153153
}
154154

155+
return randomTransformConfigWithSettings(settingsConfig, pivotConfig, latestConfig);
156+
}
157+
158+
public static TransformConfig randomTransformConfigWithSettings(
159+
SettingsConfig settingsConfig,
160+
PivotConfig pivotConfig,
161+
LatestConfig latestConfig
162+
) {
155163
return new TransformConfig(
156164
randomAlphaOfLengthBetween(1, 10),
157165
randomSourceConfig(),

‎x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,18 @@
77

88
package org.elasticsearch.xpack.transform.integration;
99

10+
import org.apache.http.HttpHost;
1011
import org.apache.http.entity.ContentType;
1112
import org.apache.http.entity.StringEntity;
1213
import org.elasticsearch.client.Request;
1314
import org.elasticsearch.client.RequestOptions;
1415
import org.elasticsearch.client.Response;
1516
import org.elasticsearch.client.ResponseException;
17+
import org.elasticsearch.client.RestClient;
18+
import org.elasticsearch.client.RestClientBuilder;
1619
import org.elasticsearch.client.WarningsHandler;
1720
import org.elasticsearch.common.Strings;
21+
import org.elasticsearch.common.settings.Settings;
1822
import org.elasticsearch.common.xcontent.support.XContentMapValues;
1923
import org.elasticsearch.xcontent.XContentBuilder;
2024
import org.junit.Before;
@@ -51,6 +55,14 @@ public class TransformPivotRestIT extends TransformRestTestCase {
5155

5256
private static boolean indicesCreated = false;
5357

58+
@Override
59+
protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException {
60+
RestClientBuilder builder = RestClient.builder(hosts);
61+
configureClient(builder, settings);
62+
builder.setStrictDeprecationMode(false);
63+
return builder.build();
64+
}
65+
5466
// preserve indices in order to reuse source indices in several test cases
5567
@Override
5668
protected boolean preserveIndicesUponCompletion() {
@@ -2707,6 +2719,54 @@ public void testExportAndImport() throws Exception {
27072719
assertThat(storedConfig, equalTo(importConfig));
27082720
}
27092721

2722+
public void testPivotWithDeprecatedMaxPageSearchSize() throws Exception {
2723+
String transformId = "deprecated_settings_pivot";
2724+
String transformIndex = "deprecated_settings_pivot_index";
2725+
setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformIndex);
2726+
2727+
final Request createTransformRequest = createRequestWithAuth(
2728+
"PUT",
2729+
getTransformEndpoint() + transformId,
2730+
BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
2731+
);
2732+
2733+
String config = Strings.format("""
2734+
{
2735+
"source": {
2736+
"index": "%s"
2737+
},
2738+
"dest": {
2739+
"index": "%s"
2740+
},
2741+
"pivot": {
2742+
"group_by": {
2743+
"every_2": {
2744+
"histogram": {
2745+
"interval": 2,
2746+
"field": "stars"
2747+
}
2748+
}
2749+
},
2750+
"aggregations": {
2751+
"avg_rating": {
2752+
"avg": {
2753+
"field": "stars"
2754+
}
2755+
}
2756+
},
2757+
"max_page_search_size": 1234
2758+
}
2759+
}""", REVIEWS_INDEX_NAME, transformIndex);
2760+
2761+
createTransformRequest.setJsonEntity(config);
2762+
Map<String, Object> createTransformResponse = entityAsMap(client().performRequest(createTransformRequest));
2763+
assertThat(createTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE));
2764+
2765+
Map<String, Object> transform = getTransformConfig(transformId, BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS);
2766+
assertNull(XContentMapValues.extractValue("pivot.max_page_search_size", transform));
2767+
assertThat(XContentMapValues.extractValue("settings.max_page_search_size", transform), equalTo(1234));
2768+
}
2769+
27102770
private void createDateNanoIndex(String indexName, int numDocs) throws IOException {
27112771
// create mapping
27122772
try (XContentBuilder builder = jsonBuilder()) {

‎x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUpdateIT.java‎

Lines changed: 0 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@
4040
public class TransformUpdateIT extends TransformRestTestCase {
4141

4242
private static final String TEST_USER_NAME = "transform_user";
43-
private static final String BASIC_AUTH_VALUE_TRANSFORM_USER = basicAuthHeaderValue(TEST_USER_NAME, TEST_PASSWORD_SECURE_STRING);
4443
private static final String TEST_ADMIN_USER_NAME_1 = "transform_admin_1";
4544
private static final String BASIC_AUTH_VALUE_TRANSFORM_ADMIN_1 = basicAuthHeaderValue(
4645
TEST_ADMIN_USER_NAME_1,
@@ -101,69 +100,6 @@ public void shutdownThreadPool() {
101100
}
102101
}
103102

104-
@SuppressWarnings("unchecked")
105-
public void testUpdateDeprecatedSettings() throws Exception {
106-
String transformId = "old_transform";
107-
String transformDest = transformId + "_idx";
108-
setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformDest);
109-
110-
final Request createTransformRequest = createRequestWithAuth(
111-
"PUT",
112-
getTransformEndpoint() + transformId,
113-
BASIC_AUTH_VALUE_TRANSFORM_ADMIN_1
114-
);
115-
String config = Strings.format("""
116-
{
117-
"dest": {
118-
"index": "%s"
119-
},
120-
"source": {
121-
"index": "%s"
122-
},
123-
"pivot": {
124-
"group_by": {
125-
"reviewer": {
126-
"terms": {
127-
"field": "user_id"
128-
}
129-
}
130-
},
131-
"aggregations": {
132-
"avg_rating": {
133-
"avg": {
134-
"field": "stars"
135-
}
136-
}
137-
},
138-
"max_page_search_size": 555
139-
}
140-
}""", transformDest, REVIEWS_INDEX_NAME);
141-
142-
createTransformRequest.setJsonEntity(config);
143-
Map<String, Object> createTransformResponse = entityAsMap(client().performRequest(createTransformRequest));
144-
assertThat(createTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE));
145-
146-
Map<String, Object> transform = getTransformConfig(transformId, BASIC_AUTH_VALUE_TRANSFORM_USER);
147-
assertThat(XContentMapValues.extractValue("pivot.max_page_search_size", transform), equalTo(555));
148-
149-
final Request updateRequest = createRequestWithAuth(
150-
"POST",
151-
getTransformEndpoint() + transformId + "/_update",
152-
BASIC_AUTH_VALUE_TRANSFORM_ADMIN_1
153-
);
154-
updateRequest.setJsonEntity("{}");
155-
156-
Map<String, Object> updateResponse = entityAsMap(client().performRequest(updateRequest));
157-
158-
assertNull(XContentMapValues.extractValue("pivot.max_page_search_size", updateResponse));
159-
assertThat(XContentMapValues.extractValue("settings.max_page_search_size", updateResponse), equalTo(555));
160-
161-
transform = getTransformConfig(transformId, BASIC_AUTH_VALUE_TRANSFORM_USER);
162-
163-
assertNull(XContentMapValues.extractValue("pivot.max_page_search_size", transform));
164-
assertThat(XContentMapValues.extractValue("settings.max_page_search_size", transform), equalTo(555));
165-
}
166-
167103
public void testUpdateTransferRights() throws Exception {
168104
updateTransferRightsTester(false);
169105
}

0 commit comments

Comments
 (0)