Skip to content

Commit 7262fb3

Browse files
prwhelanprwhelan
authored
[Transform] Auto-migrate max_page_search_size (#119348)
`max_page_search_size` had been deprecated in Pivot and migrated to Settings in ES 7.x. We are removing the key in Pivot in 9.x. It is still possible to create Pivots with `max_page_search_size`, though a deprecation is logged. To prep for its removal, this change automatically migrates `max_page_search_size` from Pivot to Settings as if the user called `_update`. - `PUT _transform` with `max_page_search_size` in Pivot will migrate the value to Settings as part of the Transform's creation. The user will receive an additional deprecation warning informing of this migration. - Existing transforms with `max_page_search_size` in Pivot will migrate the value to Settings as part of node assignment during start. Transform Messages and Elasticsearch logs will record this event so users will know what happened.
1 parent 17ece0a commit 7262fb3

File tree

14 files changed

+580
-120
lines changed

14 files changed

+580
-120
lines changed

‎docs/changelog/119348.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 119348
2+
summary: Auto-migrate `max_page_search_size`
3+
area: Transform
4+
type: enhancement
5+
issues: []

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformMessages.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,9 @@ public class TransformMessages {
8989
public static final String INVALID_ID = "Invalid {0}; ''{1}'' can contain lowercase alphanumeric (a-z and 0-9), hyphens or "
9090
+ "underscores; must start and end with alphanumeric";
9191

92+
public static final String MAX_PAGE_SEARCH_SIZE_MIGRATION =
93+
"Automatically migrating [max_page_search_size] from the pivot to the transform setting.";
94+
9295
private TransformMessages() {}
9396

9497
/**

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java‎

Lines changed: 33 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -579,24 +579,15 @@ public static TransformConfig rewriteForUpdate(final TransformConfig transformCo
579579

580580
private static TransformConfig applyRewriteForUpdate(Builder builder) {
581581
// 1. Move pivot.max_page_size_search to settings.max_page_size_search
582-
if (builder.getPivotConfig() != null && builder.getPivotConfig().getMaxPageSearchSize() != null) {
583-
584-
// find maxPageSearchSize value
585-
Integer maxPageSearchSizeDeprecated = builder.getPivotConfig().getMaxPageSearchSize();
586-
Integer maxPageSearchSize = builder.getSettings().getMaxPageSearchSize() != null
587-
? builder.getSettings().getMaxPageSearchSize()
588-
: maxPageSearchSizeDeprecated;
582+
migrateMaxPageSearchSize(builder);
589583

590-
// create a new pivot config but set maxPageSearchSize to null
591-
builder.setPivotConfig(
592-
new PivotConfig(builder.getPivotConfig().getGroupConfig(), builder.getPivotConfig().getAggregationConfig(), null)
593-
);
594-
// create new settings with maxPageSearchSize
584+
// 2. set dates_as_epoch_millis to true for transforms < 7.11 to keep BWC
585+
if (builder.getVersion() != null && builder.getVersion().before(TransformConfigVersion.V_7_11_0)) {
595586
builder.setSettings(
596587
new SettingsConfig(
597-
maxPageSearchSize,
588+
builder.getSettings().getMaxPageSearchSize(),
598589
builder.getSettings().getDocsPerSecond(),
599-
builder.getSettings().getDatesAsEpochMillis(),
590+
true,
600591
builder.getSettings().getAlignCheckpoints(),
601592
builder.getSettings().getUsePit(),
602593
builder.getSettings().getDeduceMappings(),
@@ -606,14 +597,14 @@ private static TransformConfig applyRewriteForUpdate(Builder builder) {
606597
);
607598
}
608599

609-
// 2. set dates_as_epoch_millis to true for transforms < 7.11 to keep BWC
610-
if (builder.getVersion() != null && builder.getVersion().before(TransformConfigVersion.V_7_11_0)) {
600+
// 3. set align_checkpoints to false for transforms < 7.15 to keep BWC
601+
if (builder.getVersion() != null && builder.getVersion().before(TransformConfigVersion.V_7_15_0)) {
611602
builder.setSettings(
612603
new SettingsConfig(
613604
builder.getSettings().getMaxPageSearchSize(),
614605
builder.getSettings().getDocsPerSecond(),
615-
true,
616-
builder.getSettings().getAlignCheckpoints(),
606+
builder.getSettings().getDatesAsEpochMillis(),
607+
false,
617608
builder.getSettings().getUsePit(),
618609
builder.getSettings().getDeduceMappings(),
619610
builder.getSettings().getNumFailureRetries(),
@@ -622,23 +613,41 @@ private static TransformConfig applyRewriteForUpdate(Builder builder) {
622613
);
623614
}
624615

625-
// 3. set align_checkpoints to false for transforms < 7.15 to keep BWC
626-
if (builder.getVersion() != null && builder.getVersion().before(TransformConfigVersion.V_7_15_0)) {
616+
return builder.setVersion(TransformConfigVersion.CURRENT).build();
617+
}
618+
619+
public boolean shouldAutoMigrateMaxPageSearchSize() {
620+
return getPivotConfig() != null && getPivotConfig().getMaxPageSearchSize() != null;
621+
}
622+
623+
public static Builder migrateMaxPageSearchSize(Builder builder) {
624+
if (builder.getPivotConfig() != null && builder.getPivotConfig().getMaxPageSearchSize() != null) {
625+
626+
// find maxPageSearchSize value
627+
Integer maxPageSearchSizeDeprecated = builder.getPivotConfig().getMaxPageSearchSize();
628+
Integer maxPageSearchSize = builder.getSettings().getMaxPageSearchSize() != null
629+
? builder.getSettings().getMaxPageSearchSize()
630+
: maxPageSearchSizeDeprecated;
631+
632+
// create a new pivot config but set maxPageSearchSize to null
633+
builder.setPivotConfig(
634+
new PivotConfig(builder.getPivotConfig().getGroupConfig(), builder.getPivotConfig().getAggregationConfig(), null)
635+
);
636+
// create new settings with maxPageSearchSize
627637
builder.setSettings(
628638
new SettingsConfig(
629-
builder.getSettings().getMaxPageSearchSize(),
639+
maxPageSearchSize,
630640
builder.getSettings().getDocsPerSecond(),
631641
builder.getSettings().getDatesAsEpochMillis(),
632-
false,
642+
builder.getSettings().getAlignCheckpoints(),
633643
builder.getSettings().getUsePit(),
634644
builder.getSettings().getDeduceMappings(),
635645
builder.getSettings().getNumFailureRetries(),
636646
builder.getSettings().getUnattended()
637647
)
638648
);
639649
}
640-
641-
return builder.setVersion(TransformConfigVersion.CURRENT).build();
650+
return builder;
642651
}
643652

644653
public static Builder builder() {

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,14 @@ public static TransformConfig randomTransformConfigWithSettings(SettingsConfig s
147147
latestConfig = LatestConfigTests.randomLatestConfig();
148148
}
149149

150+
return randomTransformConfigWithSettings(settingsConfig, pivotConfig, latestConfig);
151+
}
152+
153+
public static TransformConfig randomTransformConfigWithSettings(
154+
SettingsConfig settingsConfig,
155+
PivotConfig pivotConfig,
156+
LatestConfig latestConfig
157+
) {
150158
return new TransformConfig(
151159
randomAlphaOfLengthBetween(1, 10),
152160
randomSourceConfig(),

‎x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,18 @@
77

88
package org.elasticsearch.xpack.transform.integration;
99

10+
import org.apache.http.HttpHost;
1011
import org.apache.http.entity.ContentType;
1112
import org.apache.http.entity.StringEntity;
1213
import org.elasticsearch.client.Request;
1314
import org.elasticsearch.client.RequestOptions;
1415
import org.elasticsearch.client.Response;
1516
import org.elasticsearch.client.ResponseException;
17+
import org.elasticsearch.client.RestClient;
18+
import org.elasticsearch.client.RestClientBuilder;
1619
import org.elasticsearch.client.WarningsHandler;
1720
import org.elasticsearch.common.Strings;
21+
import org.elasticsearch.common.settings.Settings;
1822
import org.elasticsearch.common.xcontent.support.XContentMapValues;
1923
import org.elasticsearch.xcontent.XContentBuilder;
2024
import org.junit.Before;
@@ -51,6 +55,14 @@ public class TransformPivotRestIT extends TransformRestTestCase {
5155

5256
private static boolean indicesCreated = false;
5357

58+
@Override
59+
protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException {
60+
RestClientBuilder builder = RestClient.builder(hosts);
61+
configureClient(builder, settings);
62+
builder.setStrictDeprecationMode(false);
63+
return builder.build();
64+
}
65+
5466
// preserve indices in order to reuse source indices in several test cases
5567
@Override
5668
protected boolean preserveIndicesUponCompletion() {
@@ -2707,6 +2719,54 @@ public void testExportAndImport() throws Exception {
27072719
assertThat(storedConfig, equalTo(importConfig));
27082720
}
27092721

2722+
public void testPivotWithDeprecatedMaxPageSearchSize() throws Exception {
2723+
String transformId = "deprecated_settings_pivot";
2724+
String transformIndex = "deprecated_settings_pivot_index";
2725+
setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformIndex);
2726+
2727+
final Request createTransformRequest = createRequestWithAuth(
2728+
"PUT",
2729+
getTransformEndpoint() + transformId,
2730+
BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS
2731+
);
2732+
2733+
String config = Strings.format("""
2734+
{
2735+
"source": {
2736+
"index": "%s"
2737+
},
2738+
"dest": {
2739+
"index": "%s"
2740+
},
2741+
"pivot": {
2742+
"group_by": {
2743+
"every_2": {
2744+
"histogram": {
2745+
"interval": 2,
2746+
"field": "stars"
2747+
}
2748+
}
2749+
},
2750+
"aggregations": {
2751+
"avg_rating": {
2752+
"avg": {
2753+
"field": "stars"
2754+
}
2755+
}
2756+
},
2757+
"max_page_search_size": 1234
2758+
}
2759+
}""", REVIEWS_INDEX_NAME, transformIndex);
2760+
2761+
createTransformRequest.setJsonEntity(config);
2762+
Map<String, Object> createTransformResponse = entityAsMap(client().performRequest(createTransformRequest));
2763+
assertThat(createTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE));
2764+
2765+
Map<String, Object> transform = getTransformConfig(transformId, BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS);
2766+
assertNull(XContentMapValues.extractValue("pivot.max_page_search_size", transform));
2767+
assertThat(XContentMapValues.extractValue("settings.max_page_search_size", transform), equalTo(1234));
2768+
}
2769+
27102770
private void createDateNanoIndex(String indexName, int numDocs) throws IOException {
27112771
// create mapping
27122772
try (XContentBuilder builder = jsonBuilder()) {

‎x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformUpdateIT.java‎

Lines changed: 0 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@
4040
public class TransformUpdateIT extends TransformRestTestCase {
4141

4242
private static final String TEST_USER_NAME = "transform_user";
43-
private static final String BASIC_AUTH_VALUE_TRANSFORM_USER = basicAuthHeaderValue(TEST_USER_NAME, TEST_PASSWORD_SECURE_STRING);
4443
private static final String TEST_ADMIN_USER_NAME_1 = "transform_admin_1";
4544
private static final String BASIC_AUTH_VALUE_TRANSFORM_ADMIN_1 = basicAuthHeaderValue(
4645
TEST_ADMIN_USER_NAME_1,
@@ -101,69 +100,6 @@ public void shutdownThreadPool() {
101100
}
102101
}
103102

104-
@SuppressWarnings("unchecked")
105-
public void testUpdateDeprecatedSettings() throws Exception {
106-
String transformId = "old_transform";
107-
String transformDest = transformId + "_idx";
108-
setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformDest);
109-
110-
final Request createTransformRequest = createRequestWithAuth(
111-
"PUT",
112-
getTransformEndpoint() + transformId,
113-
BASIC_AUTH_VALUE_TRANSFORM_ADMIN_1
114-
);
115-
String config = Strings.format("""
116-
{
117-
"dest": {
118-
"index": "%s"
119-
},
120-
"source": {
121-
"index": "%s"
122-
},
123-
"pivot": {
124-
"group_by": {
125-
"reviewer": {
126-
"terms": {
127-
"field": "user_id"
128-
}
129-
}
130-
},
131-
"aggregations": {
132-
"avg_rating": {
133-
"avg": {
134-
"field": "stars"
135-
}
136-
}
137-
},
138-
"max_page_search_size": 555
139-
}
140-
}""", transformDest, REVIEWS_INDEX_NAME);
141-
142-
createTransformRequest.setJsonEntity(config);
143-
Map<String, Object> createTransformResponse = entityAsMap(client().performRequest(createTransformRequest));
144-
assertThat(createTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE));
145-
146-
Map<String, Object> transform = getTransformConfig(transformId, BASIC_AUTH_VALUE_TRANSFORM_USER);
147-
assertThat(XContentMapValues.extractValue("pivot.max_page_search_size", transform), equalTo(555));
148-
149-
final Request updateRequest = createRequestWithAuth(
150-
"POST",
151-
getTransformEndpoint() + transformId + "/_update",
152-
BASIC_AUTH_VALUE_TRANSFORM_ADMIN_1
153-
);
154-
updateRequest.setJsonEntity("{}");
155-
156-
Map<String, Object> updateResponse = entityAsMap(client().performRequest(updateRequest));
157-
158-
assertNull(XContentMapValues.extractValue("pivot.max_page_search_size", updateResponse));
159-
assertThat(XContentMapValues.extractValue("settings.max_page_search_size", updateResponse), equalTo(555));
160-
161-
transform = getTransformConfig(transformId, BASIC_AUTH_VALUE_TRANSFORM_USER);
162-
163-
assertNull(XContentMapValues.extractValue("pivot.max_page_search_size", transform));
164-
assertThat(XContentMapValues.extractValue("settings.max_page_search_size", transform), equalTo(555));
165-
}
166-
167103
public void testUpdateTransferRights() throws Exception {
168104
updateTransferRightsTester(false);
169105
}

0 commit comments

Comments
 (0)