Skip to content

Commit 3423dc8

Browse files
alex-spiesalex-spies
committed
Merge remote-tracking branch 'upstream/main' into unmute_fork_and_tsdb_tests
2 parents bccdcc5 + b404a8b commit 3423dc8

File tree

43 files changed

+438
-974
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+438
-974
lines changed

‎docs/changelog/122938.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 122938
2+
summary: Fix geoip databases index access after system feature migration (again)
3+
area: Ingest Node
4+
type: bug
5+
issues: []

‎modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,13 +123,19 @@ public void testGeoIpSystemFeaturesMigration() throws Exception {
123123

124124
// as should a normal get *
125125
assertBusy(() -> testGetStar(List.of("my-index-00001"), maybeSecurityIndex));
126+
127+
// and getting data streams
128+
assertBusy(() -> testGetDatastreams());
126129
} else {
127130
// after the upgrade, but before the migration, Kibana should work
128131
assertBusy(() -> testGetStarAsKibana(List.of("my-index-00001"), maybeSecurityIndex));
129132

130133
// as should a normal get *
131134
assertBusy(() -> testGetStar(List.of("my-index-00001"), maybeSecurityIndex));
132135

136+
// and getting data streams
137+
assertBusy(() -> testGetDatastreams());
138+
133139
// migrate the system features and give the cluster a moment to settle
134140
Request migrateSystemFeatures = new Request("POST", "/_migration/system_features");
135141
assertOK(client().performRequest(migrateSystemFeatures));
@@ -144,6 +150,9 @@ public void testGeoIpSystemFeaturesMigration() throws Exception {
144150
// as should a normal get *
145151
assertBusy(() -> testGetStar(List.of("my-index-00001"), maybeSecurityIndexReindexed));
146152

153+
// and getting data streams
154+
assertBusy(() -> testGetDatastreams());
155+
147156
Request disableDownloader = new Request("PUT", "/_cluster/settings");
148157
disableDownloader.setJsonEntity("""
149158
{"persistent": {"ingest.geoip.downloader.enabled": false}}
@@ -257,4 +266,15 @@ private void testGetStarAsKibana(List<String> indexNames, @Nullable List<String>
257266
Map<String, Object> map = responseAsMap(response);
258267
assertThat(map.keySet(), is(new HashSet<>(indexNames)));
259268
}
269+
270+
private void testGetDatastreams() throws IOException {
271+
Request getStar = new Request("GET", "_data_stream");
272+
getStar.setOptions(
273+
RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE) // we don't care about warnings, just errors
274+
);
275+
Response response = client().performRequest(getStar);
276+
assertOK(response);
277+
278+
// note: we don't actually care about the response, just that there was one and that it didn't error out on us
279+
}
260280
}

‎modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceTests.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
import org.elasticsearch.persistent.PersistentTasksCustomMetadata;
5151
import org.elasticsearch.search.SearchHit;
5252
import org.elasticsearch.search.SearchHits;
53+
import org.elasticsearch.search.SearchResponseUtils;
5354
import org.elasticsearch.test.ESTestCase;
5455
import org.elasticsearch.threadpool.TestThreadPool;
5556
import org.elasticsearch.threadpool.ThreadPool;
@@ -341,7 +342,7 @@ private String mockSearches(String databaseName, int firstChunk, int lastChunk)
341342
}
342343

343344
SearchHits hits = SearchHits.unpooled(new SearchHit[] { hit }, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1f);
344-
SearchResponse searchResponse = new SearchResponse(hits, null, null, false, null, null, 0, null, 1, 1, 0, 1L, null, null);
345+
SearchResponse searchResponse = SearchResponseUtils.successfulResponse(hits);
345346
toRelease.add(searchResponse::decRef);
346347
@SuppressWarnings("unchecked")
347348
ActionFuture<SearchResponse> actionFuture = mock(ActionFuture.class);

‎modules/reindex/src/test/java/org/elasticsearch/reindex/AsyncBulkByScrollActionTests.java‎

Lines changed: 2 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@
6868
import org.elasticsearch.rest.RestStatus;
6969
import org.elasticsearch.search.SearchHit;
7070
import org.elasticsearch.search.SearchHits;
71+
import org.elasticsearch.search.SearchResponseUtils;
7172
import org.elasticsearch.tasks.Task;
7273
import org.elasticsearch.tasks.TaskId;
7374
import org.elasticsearch.tasks.TaskManager;
@@ -574,22 +575,7 @@ protected RequestWrapper<?> buildRequest(Hit doc) {
574575
new TotalHits(0, TotalHits.Relation.EQUAL_TO),
575576
0
576577
);
577-
SearchResponse searchResponse = new SearchResponse(
578-
hits,
579-
null,
580-
null,
581-
false,
582-
false,
583-
null,
584-
1,
585-
scrollId(),
586-
5,
587-
4,
588-
0,
589-
randomLong(),
590-
null,
591-
SearchResponse.Clusters.EMPTY
592-
);
578+
SearchResponse searchResponse = SearchResponseUtils.response(hits).scrollId(scrollId()).shards(5, 4, 0).build();
593579
try {
594580
client.lastSearch.get().listener.onResponse(searchResponse);
595581

‎modules/reindex/src/test/java/org/elasticsearch/reindex/ClientScrollableHitSourceTests.java‎

Lines changed: 2 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.elasticsearch.index.reindex.ScrollableHitSource;
3131
import org.elasticsearch.search.SearchHit;
3232
import org.elasticsearch.search.SearchHits;
33+
import org.elasticsearch.search.SearchResponseUtils;
3334
import org.elasticsearch.tasks.TaskId;
3435
import org.elasticsearch.test.ESTestCase;
3536
import org.elasticsearch.threadpool.TestThreadPool;
@@ -166,22 +167,7 @@ private SearchResponse createSearchResponse() {
166167
new TotalHits(0, TotalHits.Relation.EQUAL_TO),
167168
0
168169
);
169-
return new SearchResponse(
170-
hits,
171-
null,
172-
null,
173-
false,
174-
false,
175-
null,
176-
1,
177-
randomSimpleString(random(), 1, 10),
178-
5,
179-
4,
180-
0,
181-
randomLong(),
182-
null,
183-
SearchResponse.Clusters.EMPTY
184-
);
170+
return SearchResponseUtils.response(hits).scrollId(randomSimpleString(random(), 1, 10)).shards(5, 4, 0).build();
185171
}
186172

187173
private void assertSameHits(List<? extends ScrollableHitSource.Hit> actual, SearchHit[] expected) {

‎muted-tests.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -335,6 +335,9 @@ tests:
335335
- class: org.elasticsearch.index.mapper.extras.ScaledFloatFieldMapperTests
336336
method: testBlockLoaderFromRowStrideReader
337337
issue: https://github.com/elastic/elasticsearch/issues/123126
338+
- class: org.elasticsearch.index.mapper.extras.ScaledFloatFieldMapperTests
339+
method: testBlockLoaderFromRowStrideReaderWithSyntheticSource
340+
issue: https://github.com/elastic/elasticsearch/issues/123145
338341

339342
# Examples:
340343
#

‎qa/ccs-unavailable-clusters/src/javaRestTest/java/org/elasticsearch/search/CrossClusterSearchUnavailableClusterIT.java‎

Lines changed: 3 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -12,16 +12,13 @@
1212
import org.apache.http.HttpEntity;
1313
import org.apache.http.entity.ContentType;
1414
import org.apache.http.nio.entity.NStringEntity;
15-
import org.apache.lucene.search.TotalHits;
1615
import org.elasticsearch.TransportVersion;
1716
import org.elasticsearch.action.admin.cluster.state.ClusterStateAction;
1817
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
1918
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
2019
import org.elasticsearch.action.search.SearchRequest;
21-
import org.elasticsearch.action.search.SearchResponse;
2220
import org.elasticsearch.action.search.SearchShardsRequest;
2321
import org.elasticsearch.action.search.SearchShardsResponse;
24-
import org.elasticsearch.action.search.ShardSearchFailure;
2522
import org.elasticsearch.action.search.TransportSearchAction;
2623
import org.elasticsearch.action.search.TransportSearchShardsAction;
2724
import org.elasticsearch.client.Request;
@@ -33,11 +30,11 @@
3330
import org.elasticsearch.cluster.node.DiscoveryNodes;
3431
import org.elasticsearch.cluster.node.VersionInformation;
3532
import org.elasticsearch.common.Strings;
33+
import org.elasticsearch.common.lucene.Lucene;
3634
import org.elasticsearch.common.settings.SecureString;
3735
import org.elasticsearch.common.settings.Settings;
3836
import org.elasticsearch.common.util.concurrent.EsExecutors;
3937
import org.elasticsearch.common.util.concurrent.ThreadContext;
40-
import org.elasticsearch.search.aggregations.InternalAggregations;
4138
import org.elasticsearch.test.cluster.ElasticsearchCluster;
4239
import org.elasticsearch.test.rest.ESRestTestCase;
4340
import org.elasticsearch.test.rest.ObjectPath;
@@ -102,21 +99,8 @@ private static MockTransportService startTransport(
10299
EsExecutors.DIRECT_EXECUTOR_SERVICE,
103100
SearchRequest::new,
104101
(request, channel, task) -> {
105-
var searchResponse = new SearchResponse(
106-
SearchHits.empty(new TotalHits(0, TotalHits.Relation.EQUAL_TO), Float.NaN),
107-
InternalAggregations.EMPTY,
108-
null,
109-
false,
110-
null,
111-
null,
112-
1,
113-
null,
114-
1,
115-
1,
116-
0,
117-
100,
118-
ShardSearchFailure.EMPTY_ARRAY,
119-
SearchResponse.Clusters.EMPTY
102+
var searchResponse = SearchResponseUtils.successfulResponse(
103+
SearchHits.empty(Lucene.TOTAL_HITS_EQUAL_TO_ZERO, Float.NaN)
120104
);
121105
try {
122106
channel.sendResponse(searchResponse);

‎server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import org.elasticsearch.common.regex.Regex;
1515
import org.elasticsearch.core.Nullable;
1616
import org.elasticsearch.core.Tuple;
17+
import org.elasticsearch.index.Index;
1718
import org.elasticsearch.index.IndexNotFoundException;
1819
import org.elasticsearch.indices.SystemIndices.SystemIndexAccessLevel;
1920

@@ -158,7 +159,26 @@ public static boolean isIndexVisible(
158159
if (indexAbstraction.isSystem()) {
159160
// check if it is net new
160161
if (resolver.getNetNewSystemIndexPredicate().test(indexAbstraction.getName())) {
161-
return isSystemIndexVisible(resolver, indexAbstraction);
162+
// don't give this code any particular credit for being *correct*. it's just trying to resolve a combination of
163+
// issues in a way that happens to *work*. there's probably a better way of writing things such that this won't
164+
// be necessary, but for the moment, it happens to be expedient to write things this way.
165+
166+
// unwrap the alias and re-run the function on the write index of the alias -- that is, the alias is visible if
167+
// the concrete index that it refers to is visible
168+
Index writeIndex = indexAbstraction.getWriteIndex();
169+
if (writeIndex == null) {
170+
return false;
171+
} else {
172+
return isIndexVisible(
173+
expression,
174+
selectorString,
175+
writeIndex.getName(),
176+
indicesOptions,
177+
metadata,
178+
resolver,
179+
includeDataStreams
180+
);
181+
}
162182
}
163183
}
164184

‎server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java‎

Lines changed: 2 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.elasticsearch.search.AbstractSearchTestCase;
2323
import org.elasticsearch.search.SearchHit;
2424
import org.elasticsearch.search.SearchHits;
25+
import org.elasticsearch.search.SearchResponseUtils;
2526
import org.elasticsearch.search.builder.PointInTimeBuilder;
2627
import org.elasticsearch.search.builder.SearchSourceBuilder;
2728
import org.elasticsearch.search.collapse.CollapseBuilder;
@@ -173,22 +174,7 @@ public void testFailOneItemFailsEntirePhase() throws IOException {
173174
@Override
174175
void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionListener<MultiSearchResponse> listener) {
175176
assertTrue(executedMultiSearch.compareAndSet(false, true));
176-
SearchResponse searchResponse = new SearchResponse(
177-
collapsedHits,
178-
null,
179-
null,
180-
false,
181-
null,
182-
null,
183-
1,
184-
null,
185-
1,
186-
1,
187-
0,
188-
0,
189-
ShardSearchFailure.EMPTY_ARRAY,
190-
SearchResponse.Clusters.EMPTY
191-
);
177+
SearchResponse searchResponse = SearchResponseUtils.successfulResponse(collapsedHits);
192178
ActionListener.respondAndRelease(
193179
listener,
194180
new MultiSearchResponse(

‎server/src/test/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolverTests.java‎

Lines changed: 72 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import java.util.function.Supplier;
3030

3131
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
32+
import static org.elasticsearch.indices.SystemIndices.EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY;
3233
import static org.elasticsearch.indices.SystemIndices.SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY;
3334
import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
3435
import static org.hamcrest.Matchers.contains;
@@ -218,18 +219,6 @@ public void testIsIndexVisible() {
218219
assertThat(isIndexVisible("data-stream1", "failures"), is(true));
219220
}
220221

221-
private boolean isIndexVisible(String index, String selector) {
222-
return IndexAbstractionResolver.isIndexVisible(
223-
"*",
224-
selector,
225-
index,
226-
IndicesOptions.strictExpandHidden(),
227-
metadata,
228-
indexNameExpressionResolver,
229-
true
230-
);
231-
}
232-
233222
public void testIsNetNewSystemIndexVisible() {
234223
final Settings settings = Settings.builder()
235224
.put("index.number_of_replicas", 0)
@@ -269,16 +258,71 @@ public void testIsNetNewSystemIndexVisible() {
269258
List.of(new SystemIndices.Feature("name", "description", List.of(fooDescriptor, barDescriptor)))
270259
);
271260

272-
final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
273-
threadContext.putHeader(SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY, "false");
274-
indexNameExpressionResolver = new IndexNameExpressionResolver(threadContext, systemIndices);
275-
indexAbstractionResolver = new IndexAbstractionResolver(indexNameExpressionResolver);
276-
277261
metadata = Metadata.builder().put(foo, true).put(barReindexed, true).put(other, true).build();
278262

279-
assertThat(isIndexVisible("other", "*"), is(true));
280-
assertThat(isIndexVisible(".foo", "*"), is(false));
281-
assertThat(isIndexVisible(".bar", "*"), is(false));
263+
// these indices options are for the GET _data_streams case
264+
final IndicesOptions noHiddenNoAliases = IndicesOptions.builder()
265+
.wildcardOptions(
266+
IndicesOptions.WildcardOptions.builder()
267+
.matchOpen(true)
268+
.matchClosed(true)
269+
.includeHidden(false)
270+
.resolveAliases(false)
271+
.build()
272+
)
273+
.build();
274+
275+
{
276+
final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
277+
threadContext.putHeader(SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY, "true");
278+
indexNameExpressionResolver = new IndexNameExpressionResolver(threadContext, systemIndices);
279+
indexAbstractionResolver = new IndexAbstractionResolver(indexNameExpressionResolver);
280+
281+
// this covers the GET * case -- with system access, you can see everything
282+
assertThat(isIndexVisible("other", "*"), is(true));
283+
assertThat(isIndexVisible(".foo", "*"), is(true));
284+
assertThat(isIndexVisible(".bar", "*"), is(true));
285+
286+
// but if you don't ask for hidden and aliases, you won't see hidden indices or aliases, naturally
287+
assertThat(isIndexVisible("other", "*", noHiddenNoAliases), is(true));
288+
assertThat(isIndexVisible(".foo", "*", noHiddenNoAliases), is(false));
289+
assertThat(isIndexVisible(".bar", "*", noHiddenNoAliases), is(false));
290+
}
291+
292+
{
293+
final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
294+
threadContext.putHeader(SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY, "false");
295+
indexNameExpressionResolver = new IndexNameExpressionResolver(threadContext, systemIndices);
296+
indexAbstractionResolver = new IndexAbstractionResolver(indexNameExpressionResolver);
297+
298+
// this covers the GET * case -- without system access, you can't see everything
299+
assertThat(isIndexVisible("other", "*"), is(true));
300+
assertThat(isIndexVisible(".foo", "*"), is(false));
301+
assertThat(isIndexVisible(".bar", "*"), is(false));
302+
303+
// no difference here in the datastream case, you can't see these then, either
304+
assertThat(isIndexVisible("other", "*", noHiddenNoAliases), is(true));
305+
assertThat(isIndexVisible(".foo", "*", noHiddenNoAliases), is(false));
306+
assertThat(isIndexVisible(".bar", "*", noHiddenNoAliases), is(false));
307+
}
308+
309+
{
310+
final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
311+
threadContext.putHeader(SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY, "true");
312+
threadContext.putHeader(EXTERNAL_SYSTEM_INDEX_ACCESS_CONTROL_HEADER_KEY, "some-elastic-product");
313+
indexNameExpressionResolver = new IndexNameExpressionResolver(threadContext, systemIndices);
314+
indexAbstractionResolver = new IndexAbstractionResolver(indexNameExpressionResolver);
315+
316+
// this covers the GET * case -- with product (only) access, you can't see everything
317+
assertThat(isIndexVisible("other", "*"), is(true));
318+
assertThat(isIndexVisible(".foo", "*"), is(false));
319+
assertThat(isIndexVisible(".bar", "*"), is(false));
320+
321+
// no difference here in the datastream case, you can't see these then, either
322+
assertThat(isIndexVisible("other", "*", noHiddenNoAliases), is(true));
323+
assertThat(isIndexVisible(".foo", "*", noHiddenNoAliases), is(false));
324+
assertThat(isIndexVisible(".bar", "*", noHiddenNoAliases), is(false));
325+
}
282326
}
283327

284328
private static XContentBuilder mappings() {
@@ -306,4 +350,12 @@ private List<String> resolveAbstractionsSelectorAllowed(List<String> expressions
306350
private List<String> resolveAbstractions(List<String> expressions, IndicesOptions indicesOptions, Supplier<Set<String>> mask) {
307351
return indexAbstractionResolver.resolveIndexAbstractions(expressions, indicesOptions, metadata, mask, (idx) -> true, true);
308352
}
353+
354+
private boolean isIndexVisible(String index, String selector) {
355+
return isIndexVisible(index, selector, IndicesOptions.strictExpandHidden());
356+
}
357+
358+
private boolean isIndexVisible(String index, String selector, IndicesOptions indicesOptions) {
359+
return IndexAbstractionResolver.isIndexVisible("*", selector, index, indicesOptions, metadata, indexNameExpressionResolver, true);
360+
}
309361
}

0 commit comments

Comments
 (0)