Skip to content

Commit 026135b

Browse files
authored
Merge branch 'main' into fuse_out_of_snapshot
2 parents 88d0fe3 + 9316d64 commit 026135b

File tree

22 files changed

+857
-135
lines changed

22 files changed

+857
-135
lines changed

docs/reference/enrich-processor/geoip-processor.md

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -206,14 +206,18 @@ PUT my_ip_locations
206206

207207
If you can’t [automatically update](#geoip-automatic-updates) your IP geolocation databases from the Elastic endpoint, you have a few other options:
208208

209-
* [Use a proxy endpoint](#use-proxy-geoip-endpoint)
209+
* [Use a reverse proxy endpoint](#use-proxy-geoip-endpoint)
210210
* [Use a custom endpoint](#use-custom-geoip-endpoint)
211211
* [Manually update your IP geolocation databases](#manually-update-geoip-databases)
212212

213213
$$$use-proxy-geoip-endpoint$$$
214-
**Use a proxy endpoint**
214+
**Use a reverse proxy endpoint**
215215

216-
If you can’t connect directly to the Elastic GeoIP endpoint, consider setting up a secure proxy. You can then specify the proxy endpoint URL in the [`ingest.geoip.downloader.endpoint`](#ingest-geoip-downloader-endpoint) setting of each node’s `elasticsearch.yml` file.
216+
If you can’t connect directly to the Elastic GeoIP endpoint, consider setting up a secure reverse proxy. You can then specify the reverse proxy endpoint URL in the [`ingest.geoip.downloader.endpoint`](#ingest-geoip-downloader-endpoint) setting of each node’s `elasticsearch.yml` file.
217+
218+
:::{note}
219+
True HTTP proxy support for GeoIP database downloads is not currently available in {{es}}.
220+
:::
217221

218222
In a strict setup the following domains may need to be added to the allowed domains list:
219223

docs/reference/query-languages/esql/_snippets/functions/description/count_distinct.md

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

docs/reference/query-languages/esql/kibana/definition/functions/count_distinct.json

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

docs/reference/query-languages/esql/kibana/docs/functions/count_distinct.md

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

server/src/main/java/module-info.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -491,4 +491,5 @@
491491
exports org.elasticsearch.inference.telemetry;
492492
exports org.elasticsearch.index.codec.vectors.diskbbq to org.elasticsearch.test.knn;
493493
exports org.elasticsearch.index.codec.vectors.cluster to org.elasticsearch.test.knn;
494+
exports org.elasticsearch.search.crossproject;
494495
}

server/src/main/java/org/elasticsearch/ElasticsearchException.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
import org.elasticsearch.search.aggregations.AggregationExecutionException;
4646
import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
4747
import org.elasticsearch.search.aggregations.UnsupportedAggregationOnDownsampledIndex;
48+
import org.elasticsearch.search.crossproject.NoMatchingProjectException;
4849
import org.elasticsearch.search.query.SearchTimeoutException;
4950
import org.elasticsearch.transport.TcpTransport;
5051
import org.elasticsearch.xcontent.ParseField;
@@ -79,6 +80,7 @@
7980
import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_UUID_NA_VALUE;
8081
import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
8182
import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureFieldName;
83+
import static org.elasticsearch.search.crossproject.CrossProjectIndexExpressionsRewriter.NO_MATCHING_PROJECT_EXCEPTION_VERSION;
8284

8385
/**
8486
* A base class for all elasticsearch exceptions.
@@ -2022,6 +2024,12 @@ private enum ElasticsearchExceptionHandle {
20222024
184,
20232025
TransportVersions.REMOTE_EXCEPTION,
20242026
TransportVersions.REMOTE_EXCEPTION_8_19
2027+
),
2028+
NO_MATCHING_PROJECT_EXCEPTION(
2029+
NoMatchingProjectException.class,
2030+
NoMatchingProjectException::new,
2031+
185,
2032+
NO_MATCHING_PROJECT_EXCEPTION_VERSION
20252033
);
20262034

20272035
final Class<? extends ElasticsearchException> exceptionClass;

server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstractionResolver.java

Lines changed: 92 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -48,92 +48,114 @@ public ResolvedIndexExpressions resolveIndexAbstractions(
4848
boolean includeDataStreams
4949
) {
5050
final ResolvedIndexExpressions.Builder resolvedExpressionsBuilder = ResolvedIndexExpressions.builder();
51-
5251
boolean wildcardSeen = false;
5352
for (String index : indices) {
54-
String indexAbstraction;
55-
boolean minus = false;
56-
if (index.charAt(0) == '-' && wildcardSeen) {
57-
indexAbstraction = index.substring(1);
58-
minus = true;
59-
} else {
60-
indexAbstraction = index;
61-
}
53+
wildcardSeen = resolveIndexAbstraction(
54+
resolvedExpressionsBuilder,
55+
index,
56+
indicesOptions,
57+
projectMetadata,
58+
allAuthorizedAndAvailableBySelector,
59+
isAuthorized,
60+
includeDataStreams,
61+
wildcardSeen
62+
);
63+
}
64+
return resolvedExpressionsBuilder.build();
65+
}
6266

63-
// Always check to see if there's a selector on the index expression
64-
final Tuple<String, String> expressionAndSelector = IndexNameExpressionResolver.splitSelectorExpression(indexAbstraction);
65-
final String selectorString = expressionAndSelector.v2();
66-
if (indicesOptions.allowSelectors() == false && selectorString != null) {
67-
throw new UnsupportedSelectorException(indexAbstraction);
68-
}
69-
indexAbstraction = expressionAndSelector.v1();
70-
IndexComponentSelector selector = IndexComponentSelector.getByKeyOrThrow(selectorString);
67+
private boolean resolveIndexAbstraction(
68+
ResolvedIndexExpressions.Builder resolvedExpressionsBuilder,
69+
String index,
70+
IndicesOptions indicesOptions,
71+
ProjectMetadata projectMetadata,
72+
Function<IndexComponentSelector, Set<String>> allAuthorizedAndAvailableBySelector,
73+
BiPredicate<String, IndexComponentSelector> isAuthorized,
74+
boolean includeDataStreams,
75+
boolean wildcardSeen
76+
) {
77+
String indexAbstraction;
78+
boolean minus = false;
79+
if (index.charAt(0) == '-' && wildcardSeen) {
80+
indexAbstraction = index.substring(1);
81+
minus = true;
82+
} else {
83+
indexAbstraction = index;
84+
}
85+
86+
// Always check to see if there's a selector on the index expression
87+
final Tuple<String, String> expressionAndSelector = IndexNameExpressionResolver.splitSelectorExpression(indexAbstraction);
88+
final String selectorString = expressionAndSelector.v2();
89+
if (indicesOptions.allowSelectors() == false && selectorString != null) {
90+
throw new UnsupportedSelectorException(indexAbstraction);
91+
}
92+
indexAbstraction = expressionAndSelector.v1();
93+
IndexComponentSelector selector = IndexComponentSelector.getByKeyOrThrow(selectorString);
7194

72-
// we always need to check for date math expressions
73-
indexAbstraction = IndexNameExpressionResolver.resolveDateMathExpression(indexAbstraction);
95+
// we always need to check for date math expressions
96+
indexAbstraction = IndexNameExpressionResolver.resolveDateMathExpression(indexAbstraction);
7497

75-
if (indicesOptions.expandWildcardExpressions() && Regex.isSimpleMatchPattern(indexAbstraction)) {
76-
wildcardSeen = true;
77-
final HashSet<String> resolvedIndices = new HashSet<>();
78-
for (String authorizedIndex : allAuthorizedAndAvailableBySelector.apply(selector)) {
79-
if (Regex.simpleMatch(indexAbstraction, authorizedIndex)
98+
if (indicesOptions.expandWildcardExpressions() && Regex.isSimpleMatchPattern(indexAbstraction)) {
99+
wildcardSeen = true;
100+
final HashSet<String> resolvedIndices = new HashSet<>();
101+
for (String authorizedIndex : allAuthorizedAndAvailableBySelector.apply(selector)) {
102+
if (Regex.simpleMatch(indexAbstraction, authorizedIndex)
103+
&& isIndexVisible(
104+
indexAbstraction,
105+
selectorString,
106+
authorizedIndex,
107+
indicesOptions,
108+
projectMetadata,
109+
indexNameExpressionResolver,
110+
includeDataStreams
111+
)) {
112+
resolveSelectorsAndCollect(authorizedIndex, selectorString, indicesOptions, resolvedIndices, projectMetadata);
113+
}
114+
}
115+
if (resolvedIndices.isEmpty()) {
116+
// es core honours allow_no_indices for each wildcard expression, we do the same here by throwing index not found.
117+
if (indicesOptions.allowNoIndices() == false) {
118+
throw new IndexNotFoundException(indexAbstraction);
119+
}
120+
resolvedExpressionsBuilder.addLocalExpressions(index, new HashSet<>(), SUCCESS);
121+
} else {
122+
if (minus) {
123+
resolvedExpressionsBuilder.excludeFromLocalExpressions(resolvedIndices);
124+
} else {
125+
resolvedExpressionsBuilder.addLocalExpressions(index, resolvedIndices, SUCCESS);
126+
}
127+
}
128+
} else {
129+
final HashSet<String> resolvedIndices = new HashSet<>();
130+
resolveSelectorsAndCollect(indexAbstraction, selectorString, indicesOptions, resolvedIndices, projectMetadata);
131+
if (minus) {
132+
resolvedExpressionsBuilder.excludeFromLocalExpressions(resolvedIndices);
133+
} else {
134+
final boolean authorized = isAuthorized.test(indexAbstraction, selector);
135+
if (authorized) {
136+
final boolean visible = indexExists(projectMetadata, indexAbstraction)
80137
&& isIndexVisible(
81138
indexAbstraction,
82139
selectorString,
83-
authorizedIndex,
140+
indexAbstraction,
84141
indicesOptions,
85142
projectMetadata,
86143
indexNameExpressionResolver,
87144
includeDataStreams
88-
)) {
89-
resolveSelectorsAndCollect(authorizedIndex, selectorString, indicesOptions, resolvedIndices, projectMetadata);
90-
}
91-
}
92-
if (resolvedIndices.isEmpty()) {
93-
// es core honours allow_no_indices for each wildcard expression, we do the same here by throwing index not found.
94-
if (indicesOptions.allowNoIndices() == false) {
95-
throw new IndexNotFoundException(indexAbstraction);
96-
}
97-
resolvedExpressionsBuilder.addLocalExpressions(index, new HashSet<>(), SUCCESS);
98-
} else {
99-
if (minus) {
100-
resolvedExpressionsBuilder.excludeFromLocalExpressions(resolvedIndices);
101-
} else {
102-
resolvedExpressionsBuilder.addLocalExpressions(index, resolvedIndices, SUCCESS);
103-
}
104-
}
105-
} else {
106-
final HashSet<String> resolvedIndices = new HashSet<>();
107-
resolveSelectorsAndCollect(indexAbstraction, selectorString, indicesOptions, resolvedIndices, projectMetadata);
108-
if (minus) {
109-
resolvedExpressionsBuilder.excludeFromLocalExpressions(resolvedIndices);
145+
);
146+
final LocalIndexResolutionResult result = visible ? SUCCESS : CONCRETE_RESOURCE_NOT_VISIBLE;
147+
resolvedExpressionsBuilder.addLocalExpressions(index, resolvedIndices, result);
148+
} else if (indicesOptions.ignoreUnavailable()) {
149+
// ignoreUnavailable implies that the request should not fail if an index is not authorized
150+
// so we map this expression to an empty list,
151+
resolvedExpressionsBuilder.addLocalExpressions(index, new HashSet<>(), CONCRETE_RESOURCE_UNAUTHORIZED);
110152
} else {
111-
final boolean authorized = isAuthorized.test(indexAbstraction, selector);
112-
if (authorized) {
113-
final boolean visible = indexExists(projectMetadata, indexAbstraction)
114-
&& isIndexVisible(
115-
indexAbstraction,
116-
selectorString,
117-
indexAbstraction,
118-
indicesOptions,
119-
projectMetadata,
120-
indexNameExpressionResolver,
121-
includeDataStreams
122-
);
123-
final LocalIndexResolutionResult result = visible ? SUCCESS : CONCRETE_RESOURCE_NOT_VISIBLE;
124-
resolvedExpressionsBuilder.addLocalExpressions(index, resolvedIndices, result);
125-
} else if (indicesOptions.ignoreUnavailable()) {
126-
// ignoreUnavailable implies that the request should not fail if an index is not authorized
127-
// so we map this expression to an empty list,
128-
resolvedExpressionsBuilder.addLocalExpressions(index, new HashSet<>(), CONCRETE_RESOURCE_UNAUTHORIZED);
129-
} else {
130-
// store the calculated expansion as unauthorized, it will be rejected later
131-
resolvedExpressionsBuilder.addLocalExpressions(index, resolvedIndices, CONCRETE_RESOURCE_UNAUTHORIZED);
132-
}
153+
// store the calculated expansion as unauthorized, it will be rejected later
154+
resolvedExpressionsBuilder.addLocalExpressions(index, resolvedIndices, CONCRETE_RESOURCE_UNAUTHORIZED);
133155
}
134156
}
135157
}
136-
return resolvedExpressionsBuilder.build();
158+
return wildcardSeen;
137159
}
138160

139161
private static void resolveSelectorsAndCollect(

server/src/main/java/org/elasticsearch/indices/IndicesExpressionGrouper.java

Lines changed: 2 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,8 @@
1111

1212
import org.elasticsearch.action.OriginalIndices;
1313
import org.elasticsearch.action.support.IndicesOptions;
14-
import org.elasticsearch.common.Strings;
1514

1615
import java.util.Map;
17-
import java.util.Set;
1816

1917
/**
2018
* Interface for grouping index expressions, along with IndicesOptions by cluster alias.
@@ -30,36 +28,7 @@
3028
public interface IndicesExpressionGrouper {
3129

3230
/**
33-
* @param remoteClusterNames Set of configured remote cluster names.
34-
* @param indicesOptions IndicesOptions to clarify how the index expression should be parsed/applied
35-
* @param indexExpressionCsv Multiple index expressions as CSV string (with no spaces), e.g., "logs1,logs2,cluster-a:logs1".
36-
* A single index expression is also supported.
37-
* @return Map where the key is the cluster alias (for "local" cluster, it is RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)
38-
* and the value for that cluster from the index expression is an OriginalIndices object.
31+
* See {@link org.elasticsearch.transport.RemoteClusterService#groupIndices} for details
3932
*/
40-
default Map<String, OriginalIndices> groupIndices(
41-
Set<String> remoteClusterNames,
42-
IndicesOptions indicesOptions,
43-
String indexExpressionCsv
44-
) {
45-
return groupIndices(remoteClusterNames, indicesOptions, Strings.splitStringByCommaToArray(indexExpressionCsv));
46-
}
47-
48-
/**
49-
* Same behavior as the other groupIndices, except the incoming multiple index expressions must already be
50-
* parsed into a String array.
51-
* @param remoteClusterNames Set of configured remote cluster names.
52-
* @param indicesOptions IndicesOptions to clarify how the index expressions should be parsed/applied
53-
* @param indexExpressions Multiple index expressions as string[].
54-
* @return Map where the key is the cluster alias (for "local" cluster, it is RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)
55-
* and the value for that cluster from the index expression is an OriginalIndices object.
56-
*/
57-
Map<String, OriginalIndices> groupIndices(Set<String> remoteClusterNames, IndicesOptions indicesOptions, String[] indexExpressions);
58-
59-
/**
60-
* Returns a set of currently configured remote clusters.
61-
*/
62-
default Set<String> getConfiguredClusters() {
63-
return Set.of();
64-
}
33+
Map<String, OriginalIndices> groupIndices(IndicesOptions indicesOptions, String[] indexExpressions, boolean returnLocalAll);
6534
}

0 commit comments

Comments
 (0)