Skip to content

Commit b7f0868

Browse files
jbaierajbaiera
authored
Merge branch '9.1' into backport/9.1/pr-135479
2 parents 2e49da4 + 9cdfe5d commit b7f0868

File tree

15 files changed

+295
-74
lines changed

15 files changed

+295
-74
lines changed

‎docs/changelog/135414.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 135414
2+
summary: "Change reindex to use ::es-redacted:: filtering"
3+
area: Audit
4+
type: enhancement
5+
issues: []

‎modules/reindex/src/main/java/org/elasticsearch/reindex/RestReindexAction.java‎

Lines changed: 39 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import org.elasticsearch.features.NodeFeature;
1515
import org.elasticsearch.index.reindex.ReindexAction;
1616
import org.elasticsearch.index.reindex.ReindexRequest;
17+
import org.elasticsearch.rest.FilteredRestRequest;
1718
import org.elasticsearch.rest.RestRequest;
1819
import org.elasticsearch.rest.RestRequestFilter;
1920
import org.elasticsearch.rest.Scope;
@@ -22,8 +23,10 @@
2223

2324
import java.io.IOException;
2425
import java.util.List;
26+
import java.util.Map;
2527
import java.util.Set;
2628
import java.util.function.Predicate;
29+
import java.util.stream.Collectors;
2730

2831
import static org.elasticsearch.core.TimeValue.parseTimeValue;
2932
import static org.elasticsearch.rest.RestRequest.Method.POST;
@@ -79,10 +82,43 @@ protected ReindexRequest buildRequest(RestRequest request) throws IOException {
7982
return internal;
8083
}
8184

82-
private static final Set<String> FILTERED_FIELDS = Set.of("source.remote.host.password");
83-
85+
/**
86+
* This method isn't used because we implement {@link #getFilteredRequest(RestRequest)} instead
87+
*/
8488
@Override
8589
public Set<String> getFilteredFields() {
86-
return FILTERED_FIELDS;
90+
assert false : "This method should never be called";
91+
throw new UnsupportedOperationException();
92+
}
93+
94+
@Override
95+
public RestRequest getFilteredRequest(RestRequest restRequest) {
96+
if (restRequest.hasContent()) {
97+
return new FilteredRestRequest(restRequest, Set.of()) {
98+
@Override
99+
@SuppressWarnings({ "rawtypes", "unchecked" })
100+
protected Map<String, Object> transformBody(Map<String, Object> map) {
101+
final var source = map.get("source");
102+
if (source instanceof Map sourceMap) {
103+
final var remote = sourceMap.get("remote");
104+
if (remote instanceof Map remoteMap) {
105+
remoteMap.computeIfPresent("password", (key, value) -> "::es-redacted::");
106+
remoteMap.computeIfPresent("headers", (key, value) -> {
107+
if (value instanceof Map<?, ?> headers) {
108+
return headers.entrySet()
109+
.stream()
110+
.collect(Collectors.toMap(Map.Entry::getKey, ignore -> "::es-redacted::"));
111+
} else {
112+
return null;
113+
}
114+
});
115+
}
116+
}
117+
return map;
118+
}
119+
};
120+
} else {
121+
return restRequest;
122+
}
87123
}
88124
}

‎modules/reindex/src/test/java/org/elasticsearch/reindex/RestReindexActionTests.java‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,10 @@
1111

1212
import org.elasticsearch.common.bytes.BytesArray;
1313
import org.elasticsearch.common.bytes.BytesReference;
14+
import org.elasticsearch.common.xcontent.XContentHelper;
1415
import org.elasticsearch.index.reindex.AbstractBulkByScrollRequest;
1516
import org.elasticsearch.index.reindex.ReindexRequest;
17+
import org.elasticsearch.rest.RestRequest;
1618
import org.elasticsearch.test.rest.FakeRestRequest;
1719
import org.elasticsearch.test.rest.RestActionTestCase;
1820
import org.elasticsearch.xcontent.XContentBuilder;
@@ -21,8 +23,16 @@
2123
import org.junit.Before;
2224

2325
import java.io.IOException;
26+
import java.util.List;
27+
import java.util.Map;
2428

2529
import static java.util.Collections.singletonMap;
30+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
31+
import static org.hamcrest.Matchers.aMapWithSize;
32+
import static org.hamcrest.Matchers.equalTo;
33+
import static org.hamcrest.Matchers.hasEntry;
34+
import static org.hamcrest.Matchers.hasKey;
35+
import static org.hamcrest.Matchers.notNullValue;
2636

2737
public class RestReindexActionTests extends RestActionTestCase {
2838

@@ -74,4 +84,150 @@ public void testSetScrollTimeout() throws IOException {
7484
assertEquals("10m", request.getScrollTime().toString());
7585
}
7686
}
87+
88+
public void testFilterSource() throws IOException {
89+
final FakeRestRequest.Builder requestBuilder = new FakeRestRequest.Builder(xContentRegistry());
90+
final var body = """
91+
{
92+
"source" : {
93+
"index": "photos",
94+
"remote" : {
95+
"host": "https://bugle.example.net:2400/",
96+
"username": "peter.parker",
97+
"password": "mj4ever!",
98+
"headers": {
99+
"X-Hero-Name": "spiderman"
100+
}
101+
}
102+
},
103+
"dest": {
104+
"index": "webshots"
105+
}
106+
}
107+
""";
108+
requestBuilder.withContent(new BytesArray(body), XContentType.JSON);
109+
110+
final FakeRestRequest restRequest = requestBuilder.build();
111+
ReindexRequest request = action.buildRequest(restRequest);
112+
113+
// Check that the request parsed correctly
114+
assertThat(request.getRemoteInfo().getScheme(), equalTo("https"));
115+
assertThat(request.getRemoteInfo().getHost(), equalTo("bugle.example.net"));
116+
assertThat(request.getRemoteInfo().getPort(), equalTo(2400));
117+
assertThat(request.getRemoteInfo().getUsername(), equalTo("peter.parker"));
118+
assertThat(request.getRemoteInfo().getPassword(), equalTo("mj4ever!"));
119+
assertThat(request.getRemoteInfo().getHeaders(), hasEntry("X-Hero-Name", "spiderman"));
120+
assertThat(request.getRemoteInfo().getHeaders(), aMapWithSize(1));
121+
122+
final RestRequest filtered = action.getFilteredRequest(restRequest);
123+
assertToXContentEquivalent(new BytesArray("""
124+
{
125+
"source" : {
126+
"index": "photos",
127+
"remote" : {
128+
"host": "https://bugle.example.net:2400/",
129+
"username": "peter.parker",
130+
"password": "::es-redacted::",
131+
"headers": {
132+
"X-Hero-Name": "::es-redacted::"
133+
}
134+
}
135+
},
136+
"dest": {
137+
"index": "webshots"
138+
}
139+
}
140+
"""), filtered.content(), XContentType.JSON);
141+
}
142+
143+
public void testUnfilteredSource() throws IOException {
144+
final FakeRestRequest.Builder requestBuilder = new FakeRestRequest.Builder(xContentRegistry());
145+
final var empty1 = "";
146+
final var empty2 = "{}";
147+
final var nonRemote = """
148+
{
149+
"source" : { "index": "your-index" },
150+
"dest" : { "index": "my-index" }
151+
}
152+
""";
153+
final var noCredentials = """
154+
{
155+
"source" : {
156+
"index": "remote-index",
157+
"remote" : {
158+
"host": "https://es.example.net:12345/",
159+
"headers": {}
160+
}
161+
},
162+
"dest": {
163+
"index": "my-index"
164+
}
165+
}
166+
""";
167+
for (String body : List.of(empty1, empty2, nonRemote, noCredentials)) {
168+
final BytesArray bodyAsBytes = new BytesArray(body);
169+
requestBuilder.withContent(bodyAsBytes, XContentType.JSON);
170+
final FakeRestRequest restRequest = requestBuilder.build();
171+
final RestRequest filtered = action.getFilteredRequest(restRequest);
172+
assertToXContentEquivalent(bodyAsBytes, filtered.content(), XContentType.JSON);
173+
}
174+
}
175+
176+
public void testFilteringBadlyStructureSourceIsSafe() throws IOException {
177+
final FakeRestRequest.Builder requestBuilder = new FakeRestRequest.Builder(xContentRegistry());
178+
final var remoteAsString = """
179+
{
180+
"source" : {
181+
"index": "remote-index",
182+
"remote" : "https://es.example.net:12345/"
183+
},
184+
"dest": {
185+
"index": "my-index"
186+
}
187+
}
188+
""";
189+
final var passwordAsNumber = """
190+
{
191+
"source" : {
192+
"index": "remote-index",
193+
"remote" : {
194+
"host": "https://es.example.net:12345/",
195+
"username": "skroob",
196+
"password": 12345
197+
}
198+
},
199+
"dest": {
200+
"index": "my-index"
201+
}
202+
}
203+
""";
204+
final var headersAsList = """
205+
{
206+
"source" : {
207+
"index": "remote-index",
208+
"remote" : {
209+
"host": "https://es.example.net:12345/",
210+
"headers": [ "bogus" ]
211+
}
212+
},
213+
"dest": {
214+
"index": "my-index"
215+
}
216+
}
217+
""";
218+
for (String body : List.of(remoteAsString, passwordAsNumber, headersAsList)) {
219+
final BytesArray bodyAsBytes = new BytesArray(body);
220+
requestBuilder.withContent(bodyAsBytes, XContentType.JSON);
221+
final FakeRestRequest restRequest = requestBuilder.build();
222+
223+
final RestRequest filtered = action.getFilteredRequest(restRequest);
224+
assertThat(filtered, notNullValue());
225+
226+
// We will redacted some parts of these bodies, so just check that they end up as valid JSON with the right top level fields
227+
final Map<String, Object> filteredMap = XContentHelper.convertToMap(filtered.content(), false, XContentType.JSON).v2();
228+
assertThat(filteredMap, notNullValue());
229+
assertThat(filteredMap, hasKey("source"));
230+
assertThat(filteredMap, hasKey("dest"));
231+
}
232+
}
77233
}

‎server/src/internalClusterTest/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,8 @@ public void testElectMasterWithLatestVersion() throws Exception {
147147
isolateAllNodes.stopDisrupting();
148148

149149
awaitMasterNode();
150-
final ClusterState state = clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT).get().getState();
150+
final var masterNodeAdminClient = client(internalCluster().getMasterName()).admin().cluster();
151+
final ClusterState state = masterNodeAdminClient.prepareState(TEST_REQUEST_TIMEOUT).get().getState();
151152
if (state.metadata().getProject().hasIndex("test") == false) {
152153
fail("index 'test' was lost. current cluster state: " + state);
153154
}

‎server/src/internalClusterTest/java/org/elasticsearch/index/engine/MergeWithFailureIT.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -321,7 +321,7 @@ public void testFailedMergeDeadlock() throws Exception {
321321
ensureRed(indexName);
322322

323323
// verify that the shard store is effectively closed
324-
assertTrue(plugin.shardStoreClosedListener.isDone());
324+
safeGet(plugin.shardStoreClosedListener);
325325

326326
if (closingThread != null) {
327327
closingThread.join();

‎server/src/main/java/org/elasticsearch/TransportVersions.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -192,7 +192,6 @@ static TransportVersion def(int id) {
192192
public static final TransportVersion ML_INFERENCE_ELASTIC_RERANK_ADDED_8_19 = def(8_841_0_48);
193193
public static final TransportVersion NONE_CHUNKING_STRATEGY_8_19 = def(8_841_0_49);
194194
public static final TransportVersion IDP_CUSTOM_SAML_ATTRIBUTES_ALLOW_LIST_8_19 = def(8_841_0_50);
195-
public static final TransportVersion SETTINGS_IN_DATA_STREAMS_8_19 = def(8_841_0_51);
196195
public static final TransportVersion INITIAL_ELASTICSEARCH_9_0 = def(9_000_0_00);
197196
public static final TransportVersion REMOVE_SNAPSHOT_FAILURES_90 = def(9_000_0_01);
198197
public static final TransportVersion TRANSPORT_STATS_HANDLING_TIME_REQUIRED_90 = def(9_000_0_02);
@@ -263,7 +262,6 @@ static TransportVersion def(int id) {
263262
public static final TransportVersion SYNONYMS_REFRESH_PARAM = def(9_060_0_00);
264263
public static final TransportVersion DOC_FIELDS_AS_LIST = def(9_061_0_00);
265264
public static final TransportVersion DENSE_VECTOR_OFF_HEAP_STATS = def(9_062_00_0);
266-
public static final TransportVersion SETTINGS_IN_DATA_STREAMS = def(9_064_0_00);
267265
public static final TransportVersion INTRODUCE_FAILURES_LIFECYCLE = def(9_065_0_00);
268266
public static final TransportVersion PROJECT_METADATA_SETTINGS = def(9_066_0_00);
269267
public static final TransportVersion AGGREGATE_METRIC_DOUBLE_BLOCK = def(9_067_0_00);
@@ -280,7 +278,6 @@ static TransportVersion def(int id) {
280278
public static final TransportVersion ML_INFERENCE_HUGGING_FACE_CHAT_COMPLETION_ADDED = def(9_078_0_00);
281279
public static final TransportVersion NODES_STATS_SUPPORTS_MULTI_PROJECT = def(9_079_0_00);
282280
public static final TransportVersion ML_INFERENCE_HUGGING_FACE_RERANK_ADDED = def(9_080_0_00);
283-
public static final TransportVersion SETTINGS_IN_DATA_STREAMS_DRY_RUN = def(9_081_0_00);
284281
public static final TransportVersion ML_INFERENCE_SAGEMAKER_CHAT_COMPLETION = def(9_082_0_00);
285282
public static final TransportVersion ML_INFERENCE_VERTEXAI_CHATCOMPLETION_ADDED = def(9_083_0_00);
286283
public static final TransportVersion INFERENCE_CUSTOM_SERVICE_ADDED = def(9_084_0_00);

‎server/src/main/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsAction.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99

1010
package org.elasticsearch.action.datastreams;
1111

12-
import org.elasticsearch.TransportVersions;
12+
import org.elasticsearch.TransportVersion;
1313
import org.elasticsearch.action.ActionResponse;
1414
import org.elasticsearch.action.ActionType;
1515
import org.elasticsearch.action.IndicesRequest;
@@ -41,6 +41,8 @@ public class UpdateDataStreamSettingsAction extends ActionType<UpdateDataStreamS
4141
public static final String NAME = "indices:admin/data_stream/settings/update";
4242
public static final UpdateDataStreamSettingsAction INSTANCE = new UpdateDataStreamSettingsAction();
4343

44+
private static final TransportVersion SETTINGS_IN_DATA_STREAMS = TransportVersion.fromName("settings_in_data_streams");
45+
4446
public UpdateDataStreamSettingsAction() {
4547
super(NAME);
4648
}
@@ -83,8 +85,7 @@ public Request(StreamInput in) throws IOException {
8385
super(in);
8486
this.dataStreamNames = in.readStringArray();
8587
this.settings = Settings.readSettingsFromStream(in);
86-
if (in.getTransportVersion().onOrAfter(TransportVersions.SETTINGS_IN_DATA_STREAMS_DRY_RUN)
87-
|| in.getTransportVersion().isPatchFrom(TransportVersions.SETTINGS_IN_DATA_STREAMS_8_19)) {
88+
if (in.getTransportVersion().supports(SETTINGS_IN_DATA_STREAMS)) {
8889
this.dryRun = in.readBoolean();
8990
} else {
9091
this.dryRun = false;
@@ -96,8 +97,7 @@ public void writeTo(StreamOutput out) throws IOException {
9697
super.writeTo(out);
9798
out.writeStringArray(dataStreamNames);
9899
settings.writeTo(out);
99-
if (out.getTransportVersion().onOrAfter(TransportVersions.SETTINGS_IN_DATA_STREAMS_DRY_RUN)
100-
|| out.getTransportVersion().isPatchFrom(TransportVersions.SETTINGS_IN_DATA_STREAMS_8_19)) {
100+
if (out.getTransportVersion().supports(SETTINGS_IN_DATA_STREAMS)) {
101101
out.writeBoolean(dryRun);
102102
}
103103
}

‎server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ public final class DataStream implements SimpleDiffable<DataStream>, ToXContentO
8484

8585
private static final Logger LOGGER = LogManager.getLogger(DataStream.class);
8686

87+
private static final TransportVersion SETTINGS_IN_DATA_STREAMS = TransportVersion.fromName("settings_in_data_streams");
8788
private static final TransportVersion MAPPINGS_IN_DATA_STREAMS = TransportVersion.fromName("mappings_in_data_streams");
8889

8990
public static final NodeFeature DATA_STREAM_FAILURE_STORE_FEATURE = new NodeFeature("data_stream.failure_store");
@@ -306,8 +307,7 @@ public static DataStream read(StreamInput in) throws IOException {
306307
dataStreamOptions = failureStoreEnabled ? DataStreamOptions.FAILURE_STORE_ENABLED : null;
307308
}
308309
final Settings settings;
309-
if (in.getTransportVersion().onOrAfter(TransportVersions.SETTINGS_IN_DATA_STREAMS)
310-
|| in.getTransportVersion().isPatchFrom(TransportVersions.SETTINGS_IN_DATA_STREAMS_8_19)) {
310+
if (in.getTransportVersion().supports(SETTINGS_IN_DATA_STREAMS)) {
311311
settings = Settings.readSettingsFromStream(in);
312312
} else {
313313
settings = Settings.EMPTY;
@@ -1391,8 +1391,7 @@ public void writeTo(StreamOutput out) throws IOException {
13911391
if (out.getTransportVersion().onOrAfter(DataStream.ADD_DATA_STREAM_OPTIONS_VERSION)) {
13921392
out.writeOptionalWriteable(dataStreamOptions.isEmpty() ? null : dataStreamOptions);
13931393
}
1394-
if (out.getTransportVersion().onOrAfter(TransportVersions.SETTINGS_IN_DATA_STREAMS)
1395-
|| out.getTransportVersion().isPatchFrom(TransportVersions.SETTINGS_IN_DATA_STREAMS_8_19)) {
1394+
if (out.getTransportVersion().supports(SETTINGS_IN_DATA_STREAMS)) {
13961395
settings.writeTo(out);
13971396
}
13981397
if (out.getTransportVersion().supports(MAPPINGS_IN_DATA_STREAMS)) {

0 commit comments

Comments
 (0)