Skip to content

Commit f69485b

Browse files
przemekwitekprzemekwitek
authored
[Transform] Surface script deprecation warnings in deprecation info API (#84040)
1 parent 6cfae7f commit f69485b

File tree

7 files changed

+252
-17
lines changed

7 files changed

+252
-17
lines changed

‎server/src/main/java/org/elasticsearch/client/OriginSettingClient.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,16 @@
2626
public final class OriginSettingClient extends FilterClient {
2727

2828
private final String origin;
29+
private final boolean preserveResponseHeaders;
2930

3031
public OriginSettingClient(Client in, String origin) {
32+
this(in, origin, false);
33+
}
34+
35+
public OriginSettingClient(Client in, String origin, boolean preserveResponseHeaders) {
3136
super(in);
3237
this.origin = origin;
38+
this.preserveResponseHeaders = preserveResponseHeaders;
3339
}
3440

3541
@Override
@@ -38,7 +44,9 @@ protected <Request extends ActionRequest, Response extends ActionResponse> void
3844
Request request,
3945
ActionListener<Response> listener
4046
) {
41-
final Supplier<ThreadContext.StoredContext> supplier = in().threadPool().getThreadContext().newRestorableContext(false);
47+
final Supplier<ThreadContext.StoredContext> supplier = in().threadPool()
48+
.getThreadContext()
49+
.newRestorableContext(preserveResponseHeaders);
4250
try (ThreadContext.StoredContext ignore = in().threadPool().getThreadContext().stashWithOrigin(origin)) {
4351
super.doExecute(action, request, new ContextPreservingActionListener<>(supplier, listener));
4452
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformDeprecations.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ public class TransformDeprecations {
1515

1616
public static final String AGGS_BREAKING_CHANGES_URL = "https://ela.st/es-deprecation-8-transform-aggregation-options";
1717

18+
public static final String PAINLESS_BREAKING_CHANGES_URL = "https://ela.st/es-deprecation-8-transform-painless-options";
19+
1820
public static final String ACTION_UPGRADE_TRANSFORMS_API =
1921
"This transform configuration uses obsolete syntax which will be unsupported after the next upgrade. "
2022
+ "Use [/_transform/_upgrade] to upgrade all transforms to the latest format.";

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransformDeprecationChecker.java‎

Lines changed: 87 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -7,21 +7,37 @@
77

88
package org.elasticsearch.xpack.deprecation;
99

10+
import org.apache.logging.log4j.LogManager;
11+
import org.apache.logging.log4j.Logger;
1012
import org.elasticsearch.action.ActionListener;
1113
import org.elasticsearch.cluster.metadata.Metadata;
14+
import org.elasticsearch.common.logging.HeaderWarning;
1215
import org.elasticsearch.common.settings.Settings;
16+
import org.elasticsearch.common.util.concurrent.CountDown;
17+
import org.elasticsearch.core.TimeValue;
18+
import org.elasticsearch.xpack.core.ClientHelper;
1319
import org.elasticsearch.xpack.core.action.util.PageParams;
1420
import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
21+
import org.elasticsearch.xpack.core.transform.TransformDeprecations;
22+
import org.elasticsearch.xpack.core.transform.TransformField;
1523
import org.elasticsearch.xpack.core.transform.action.GetTransformAction;
24+
import org.elasticsearch.xpack.core.transform.action.ValidateTransformAction;
1625
import org.elasticsearch.xpack.core.transform.transforms.TransformConfig;
1726

1827
import java.util.ArrayList;
28+
import java.util.Collection;
29+
import java.util.Collections;
1930
import java.util.List;
31+
import java.util.concurrent.ConcurrentLinkedQueue;
32+
33+
import static java.util.stream.Collectors.toList;
2034

2135
public class TransformDeprecationChecker implements DeprecationChecker {
2236

2337
public static final String TRANSFORM_DEPRECATION_KEY = "transform_settings";
2438

39+
private static final Logger logger = LogManager.getLogger(TransformDeprecationChecker.class);
40+
2541
@Override
2642
public boolean enabled(Settings settings) {
2743
// always enabled
@@ -32,7 +48,7 @@ public boolean enabled(Settings settings) {
3248
public void check(Components components, ActionListener<CheckResult> deprecationIssueListener) {
3349

3450
PageParams startPage = new PageParams(0, PageParams.DEFAULT_SIZE);
35-
List<DeprecationIssue> issues = new ArrayList<>();
51+
Collection<DeprecationIssue> issues = new ConcurrentLinkedQueue<>();
3652
recursiveGetTransformsAndCollectDeprecations(
3753
components,
3854
issues,
@@ -51,25 +67,82 @@ public String getName() {
5167

5268
private void recursiveGetTransformsAndCollectDeprecations(
5369
Components components,
54-
List<DeprecationIssue> issues,
70+
Collection<DeprecationIssue> issues,
5571
PageParams page,
5672
ActionListener<List<DeprecationIssue>> listener
5773
) {
5874
final GetTransformAction.Request request = new GetTransformAction.Request(Metadata.ALL);
5975
request.setPageParams(page);
6076
request.setAllowNoResources(true);
6177

62-
components.client().execute(GetTransformAction.INSTANCE, request, ActionListener.wrap(getTransformResponse -> {
63-
for (TransformConfig config : getTransformResponse.getTransformConfigurations()) {
64-
issues.addAll(config.checkForDeprecations(components.xContentRegistry()));
65-
}
66-
if (getTransformResponse.getCount() >= (page.getFrom() + page.getSize())) {
67-
PageParams nextPage = new PageParams(page.getFrom() + page.getSize(), PageParams.DEFAULT_SIZE);
68-
recursiveGetTransformsAndCollectDeprecations(components, issues, nextPage, listener);
69-
} else {
70-
listener.onResponse(issues);
71-
}
72-
73-
}, listener::onFailure));
78+
ClientHelper.executeAsyncWithOrigin(
79+
components.client(),
80+
ClientHelper.DEPRECATION_ORIGIN,
81+
GetTransformAction.INSTANCE,
82+
request,
83+
ActionListener.wrap(getTransformResponse -> {
84+
final int numberOfTransforms = getTransformResponse.getTransformConfigurations().size();
85+
Runnable processNextPage = () -> {
86+
if (getTransformResponse.getCount() >= (page.getFrom() + page.getSize())) {
87+
PageParams nextPage = new PageParams(page.getFrom() + page.getSize(), PageParams.DEFAULT_SIZE);
88+
recursiveGetTransformsAndCollectDeprecations(components, issues, nextPage, listener);
89+
} else {
90+
listener.onResponse(new ArrayList<>(issues));
91+
}
92+
};
93+
if (numberOfTransforms == 0) {
94+
processNextPage.run();
95+
return;
96+
}
97+
final CountDown numberOfResponsesToProcess = new CountDown(numberOfTransforms);
98+
for (TransformConfig config : getTransformResponse.getTransformConfigurations()) {
99+
issues.addAll(config.checkForDeprecations(components.xContentRegistry()));
100+
101+
ValidateTransformAction.Request validateTransformRequest = new ValidateTransformAction.Request(
102+
config,
103+
false,
104+
TimeValue.timeValueSeconds(30)
105+
);
106+
ActionListener<ValidateTransformAction.Response> validateTransformListener = ActionListener.wrap(
107+
validateTransformResponse -> {
108+
List<String> warningHeaders = components.client()
109+
.threadPool()
110+
.getThreadContext()
111+
.getResponseHeaders()
112+
.get("Warning");
113+
if (warningHeaders != null) {
114+
issues.addAll(
115+
warningHeaders.stream()
116+
.map(warningHeader -> createDeprecationIssue(config.getId(), warningHeader))
117+
.collect(toList())
118+
);
119+
}
120+
if (numberOfResponsesToProcess.countDown()) {
121+
processNextPage.run();
122+
}
123+
},
124+
e -> {
125+
logger.warn("An exception occurred while gathering deprecation warnings for transform", e);
126+
if (numberOfResponsesToProcess.countDown()) {
127+
processNextPage.run();
128+
}
129+
}
130+
);
131+
132+
components.client().execute(ValidateTransformAction.INSTANCE, validateTransformRequest, validateTransformListener);
133+
}
134+
}, listener::onFailure)
135+
);
136+
}
137+
138+
private static DeprecationIssue createDeprecationIssue(String transformId, String warningHeader) {
139+
return new DeprecationIssue(
140+
DeprecationIssue.Level.WARNING,
141+
"Transform [" + transformId + "]: " + HeaderWarning.extractWarningValueFromWarningHeader(warningHeader, true),
142+
TransformDeprecations.PAINLESS_BREAKING_CHANGES_URL,
143+
null,
144+
false,
145+
Collections.singletonMap(TransformField.TRANSFORM_ID, transformId)
146+
);
74147
}
75148
}

‎x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/TransportDeprecationInfoAction.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ protected final void masterOperation(
122122
DeprecationChecker.Components components = new DeprecationChecker.Components(
123123
xContentRegistry,
124124
settings,
125-
new OriginSettingClient(client, ClientHelper.DEPRECATION_ORIGIN),
125+
new OriginSettingClient(client, ClientHelper.DEPRECATION_ORIGIN, true),
126126
state
127127
);
128128
pluginSettingIssues(PLUGIN_CHECKERS, components, ActionListener.wrap(deprecationIssues -> {

‎x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/TransformDeprecationCheckerTests.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.deprecation;
9+
10+
import org.elasticsearch.common.settings.Settings;
11+
import org.elasticsearch.test.ESTestCase;
12+
13+
import static org.hamcrest.Matchers.is;
14+
15+
public class TransformDeprecationCheckerTests extends ESTestCase {
16+
17+
public void testEnabled() {
18+
TransformDeprecationChecker transformDeprecationChecker = new TransformDeprecationChecker();
19+
assertThat(transformDeprecationChecker.enabled(Settings.EMPTY), is(true));
20+
}
21+
}

‎x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformDeprecationIT.java‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.transform.integration;
9+
10+
import org.apache.http.util.EntityUtils;
11+
import org.elasticsearch.client.Request;
12+
import org.elasticsearch.client.RequestOptions;
13+
import org.elasticsearch.client.Response;
14+
import org.elasticsearch.client.WarningsHandler;
15+
import org.junit.Before;
16+
17+
import java.io.IOException;
18+
import java.util.Arrays;
19+
20+
import static org.hamcrest.Matchers.allOf;
21+
import static org.hamcrest.Matchers.containsString;
22+
import static org.hamcrest.Matchers.not;
23+
24+
public class TransformDeprecationIT extends TransformRestTestCase {
25+
26+
private static final String TEST_USER_NAME = "transform_admin_plus_data";
27+
private static final String DATA_ACCESS_ROLE = "test_data_access";
28+
29+
private static boolean indicesCreated = false;
30+
31+
// preserve indices in order to reuse source indices in several test cases
32+
@Override
33+
protected boolean preserveIndicesUponCompletion() {
34+
return true;
35+
}
36+
37+
@Before
38+
public void createIndexes() throws IOException {
39+
setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME);
40+
setupUser(TEST_USER_NAME, Arrays.asList("transform_admin", DATA_ACCESS_ROLE));
41+
42+
// it's not possible to run it as @BeforeClass as clients aren't initialized then, so we need this little hack
43+
if (indicesCreated) {
44+
return;
45+
}
46+
createReviewsIndex();
47+
indicesCreated = true;
48+
}
49+
50+
public void testDeprecationInfo() throws Exception {
51+
{
52+
Request deprecationInfoRequest = new Request("GET", "_migration/deprecations");
53+
Response deprecationInfoResponse = client().performRequest(deprecationInfoRequest);
54+
assertThat(EntityUtils.toString(deprecationInfoResponse.getEntity()), not(containsString("Use of the joda time method")));
55+
}
56+
{
57+
String transformId = "script_deprecated_syntax";
58+
Request createTransformRequest = new Request("PUT", getTransformEndpoint() + transformId);
59+
// We need this as creation of transform with deprecated script syntax triggers warnings
60+
createTransformRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE));
61+
String config = createTransformConfig(REVIEWS_INDEX_NAME, transformId);
62+
createTransformRequest.setJsonEntity(config);
63+
client().performRequest(createTransformRequest);
64+
}
65+
{
66+
Request deprecationInfoRequest = new Request("GET", "_migration/deprecations");
67+
deprecationInfoRequest.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE));
68+
Response deprecationInfoResponse = client().performRequest(deprecationInfoRequest);
69+
assertThat(
70+
EntityUtils.toString(deprecationInfoResponse.getEntity()),
71+
allOf(
72+
containsString("Use of the joda time method [getMillis()] is deprecated. Use [toInstant().toEpochMilli()] instead."),
73+
containsString("Use of the joda time method [getEra()] is deprecated. Use [get(ChronoField.ERA)] instead.")
74+
)
75+
);
76+
}
77+
}
78+
79+
private static String createTransformConfig(String sourceIndex, String destinationIndex) {
80+
return "{"
81+
+ " \"source\": {"
82+
+ " \"index\": \""
83+
+ sourceIndex
84+
+ "\","
85+
+ " \"runtime_mappings\": {"
86+
+ " \"timestamp-5m\": {"
87+
+ " \"type\": \"date\","
88+
+ " \"script\": {"
89+
// We don't use "era" for anything in this script. This is solely to generate the deprecation warning.
90+
+ " \"source\": \"def era = doc['timestamp'].value.era; emit(doc['timestamp'].value.millis)\""
91+
+ " }"
92+
+ " }"
93+
+ " }"
94+
+ " },"
95+
+ " \"dest\": {"
96+
+ " \"index\": \""
97+
+ destinationIndex
98+
+ "\""
99+
+ " },"
100+
+ " \"pivot\": {"
101+
+ " \"group_by\": {"
102+
+ " \"timestamp\": {"
103+
+ " \"date_histogram\": {"
104+
+ " \"field\": \"timestamp-5m\","
105+
+ " \"calendar_interval\": \"1m\""
106+
+ " }"
107+
+ " }"
108+
+ " },"
109+
+ " \"aggregations\": {"
110+
+ " \"bytes.avg\": {"
111+
+ " \"avg\": {"
112+
+ " \"field\": \"bytes\""
113+
+ " }"
114+
+ " },"
115+
+ " \"millis\": {"
116+
+ " \"scripted_metric\": {"
117+
+ " \"init_script\": \"state.m = 0\","
118+
+ " \"map_script\": \"state.m = doc['timestamp'].value.millis;\","
119+
+ " \"combine_script\": \"return state.m;\","
120+
+ " \"reduce_script\": \"def last = 0; for (s in states) {last = s;} return last;\""
121+
+ " }"
122+
+ " }"
123+
+ " }"
124+
+ " }"
125+
+ "}";
126+
}
127+
}

‎x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/common/AbstractCompositeAggFunction.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,11 @@ private SearchRequest buildSearchRequest(SourceConfig sourceConfig, Map<String,
180180
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(sourceConfig.getQueryConfig().getQuery())
181181
.runtimeMappings(sourceConfig.getRuntimeMappings());
182182
buildSearchQuery(sourceBuilder, null, pageSize);
183-
return new SearchRequest(sourceConfig.getIndex()).source(sourceBuilder).indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN);
183+
return new SearchRequest(sourceConfig.getIndex()).indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN)
184+
.source(sourceBuilder)
185+
// This is done so that 2 consecutive queries return the same warnings in response headers.
186+
// If the request is cached, the second time it is called the response headers are not preserved.
187+
.requestCache(false);
184188
}
185189

186190
@Override

0 commit comments

Comments
 (0)