Skip to content

Commit 64f954a

Browse files
Merge remote-tracking branch 'upstream/main' into ES-10037-wordsmith-autosharding-logs
2 parents d90523e + a152b4e commit 64f954a

File tree

41 files changed

+1508
-99
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+1508
-99
lines changed

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/dra/DraResolvePlugin.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,16 @@ private void configureDraRepository(
7373
patternLayout.artifact(
7474
String.format("/%s/%s/downloads/%s/[module]/[module]-[revision]-[classifier].[ext]", draKey, buildId, draKey)
7575
);
76+
77+
if ("beats".equals(draKey)) {
78+
// we don't have a good pattern here for beats fips specific images
79+
patternLayout.artifact(
80+
String.format("/%s/%s/downloads/%s/metricbeat/[module]-[revision]-[classifier].[ext]", draKey, buildId, draKey)
81+
);
82+
patternLayout.artifact(
83+
String.format("/%s/%s/downloads/%s/filebeat/[module]-[revision]-[classifier].[ext]", draKey, buildId, draKey)
84+
);
85+
}
7686
});
7787
repo.metadataSources(metadataSources -> metadataSources.artifact());
7888
repo.content(repositoryContentDescriptor -> repositoryContentDescriptor.includeVersionByRegex(".*", ".*", includeVersionRegex));

distribution/docker/build.gradle

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,10 +57,14 @@ if (useDra == false) {
5757
patternLayout {
5858
if (VersionProperties.isElasticsearchSnapshot()) {
5959
artifact '/[organization]/[revision]/downloads/[organization]/[module]/[module]-[revision]-[classifier].[ext]'
60+
artifact '/[organization]/[revision]/downloads/[organization]/filebeat/[module]-[revision]-[classifier].[ext]'
61+
artifact '/[organization]/[revision]/downloads/[organization]/metricbeat/[module]-[revision]-[classifier].[ext]'
6062
} else {
6163
// When building locally we always use snapshot artifacts even if passing `-Dbuild.snapshot=false`.
6264
// Release builds are always done with a local repo.
6365
artifact '/[organization]/[revision]-SNAPSHOT/downloads/[organization]/[module]/[module]-[revision]-SNAPSHOT-[classifier].[ext]'
66+
artifact '/[organization]/[revision]-SNAPSHOT/downloads/[organization]/filebeat/[module]-[revision]-SNAPSHOT-[classifier].[ext]'
67+
artifact '/[organization]/[revision]-SNAPSHOT/downloads/[organization]/metricbeat/[module]-[revision]-SNAPSHOT-[classifier].[ext]'
6468
}
6569
}
6670
}
@@ -93,9 +97,13 @@ configurations {
9397
tini
9498
allPlugins
9599
filebeat_aarch64
100+
filebeat_fips_aarch64
96101
filebeat_x86_64
102+
filebeat_fips_x86_64
97103
metricbeat_aarch64
104+
metricbeat_fips_aarch64
98105
metricbeat_x86_64
106+
metricbeat_fips_x86_64
99107
fips
100108
}
101109

@@ -111,8 +119,15 @@ dependencies {
111119
allPlugins project(path: ':plugins', configuration: 'allPlugins')
112120
filebeat_aarch64 "beats:filebeat:${VersionProperties.elasticsearch}:[email protected]"
113121
filebeat_x86_64 "beats:filebeat:${VersionProperties.elasticsearch}:[email protected]"
122+
filebeat_fips_aarch64 "beats:filebeat-fips:${VersionProperties.elasticsearch}:[email protected]"
123+
filebeat_fips_x86_64 "beats:filebeat-fips:${VersionProperties.elasticsearch}:[email protected]"
124+
114125
metricbeat_aarch64 "beats:metricbeat:${VersionProperties.elasticsearch}:[email protected]"
115126
metricbeat_x86_64 "beats:metricbeat:${VersionProperties.elasticsearch}:[email protected]"
127+
128+
metricbeat_fips_aarch64 "beats:metricbeat-fips:${VersionProperties.elasticsearch}:[email protected]"
129+
metricbeat_fips_x86_64 "beats:metricbeat-fips:${VersionProperties.elasticsearch}:[email protected]"
130+
116131
fips "org.bouncycastle:bc-fips:1.0.2.5"
117132
fips "org.bouncycastle:bctls-fips:1.0.19"
118133
}
@@ -301,8 +316,8 @@ void addBuildDockerContextTask(Architecture architecture, DockerBase base) {
301316
boolean includeBeats = VersionProperties.isElasticsearchSnapshot() == true || buildId != null || useDra
302317

303318
if (includeBeats) {
304-
from configurations.getByName("filebeat_${architecture.classifier}")
305-
from configurations.getByName("metricbeat_${architecture.classifier}")
319+
from configurations.getByName("filebeat_fips_${architecture.classifier}")
320+
from configurations.getByName("metricbeat_fips_${architecture.classifier}")
306321
// For some reason, the artifact name can differ depending on what repository we used.
307322
rename ~/((?:file|metric)beat)-.*\.tar\.gz$/, "\$1-${VersionProperties.elasticsearch}.tar.gz"
308323
}

docs/changelog/126296.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 126296
2+
summary: Fail with 500 not 400 for `ValueExtractor` bugs
3+
area: ES|QL
4+
type: bug
5+
issues: []

docs/reference/elasticsearch/index-settings/slow-log.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,7 @@ If you aren’t sure how to start investigating traffic issues, consider enablin
216216

217217
Slow log thresholds being met does not guarantee cluster performance issues. In the event that symptoms are noticed, slow logs can provide helpful data to diagnose upstream traffic patterns or sources to resolve client-side issues. For example, you can use data included in `X-Opaque-ID`, the `_source` request body, or `user.*` fields to identify the source of your issue. This is similar to troubleshooting [live expensive tasks](docs-content://troubleshoot/elasticsearch/task-queue-backlog.md).
218218

219-
If you’re experiencing search performance issues, then you might also consider investigating searches flagged for their query durations using the [profile API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-profile.html). You can then use the profiled query to investigate optimization options using the [query profiler](docs-content://explore-analyze/query-filter/tools/search-profiler.md). This type of investigation should usually take place in a non-production environment.
219+
If you’re experiencing search performance issues, then you might also consider investigating searches flagged for their query durations using the [profile API](/reference/elasticsearch/rest-apis/search-profile.md). You can then use the profiled query to investigate optimization options using the [query profiler](docs-content://explore-analyze/query-filter/tools/search-profiler.md). This type of investigation should usually take place in a non-production environment.
220220

221221
Slow logging checks each event against the reporting threshold when the event is complete. This means that it can’t report if events trigger [circuit breaker errors](docs-content://troubleshoot/elasticsearch/circuit-breaker-errors.md). If suspect circuit breaker errors, then you should also consider enabling [audit logging](docs-content://deploy-manage/security/logging-configuration/enabling-audit-logs.md), which logs events before they are executed.
222222

docs/reference/elasticsearch/mapping-reference/geo-point.md

Lines changed: 71 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,23 @@ mapped_pages:
99

1010
Fields of type `geo_point` accept latitude-longitude pairs, which can be used:
1111

12-
* to find geopoints within a [bounding box](/reference/query-languages/query-dsl/query-dsl-geo-bounding-box-query.md), within a certain [distance](/reference/query-languages/query-dsl/query-dsl-geo-distance-query.md) of a central point, or within a [`geo_shape` query](/reference/query-languages/query-dsl/query-dsl-geo-shape-query.md) (for example, points in a polygon).
12+
* to find geopoints within a [bounding box](/reference/query-languages/query-dsl/query-dsl-geo-bounding-box-query.md),
13+
within a certain [distance](/reference/query-languages/query-dsl/query-dsl-geo-distance-query.md) of a central point,
14+
or within a [`geo_shape` query](/reference/query-languages/query-dsl/query-dsl-geo-shape-query.md) (for example, points in a polygon).
1315
* to aggregate documents by [distance](/reference/aggregations/search-aggregations-bucket-geodistance-aggregation.md) from a central point.
14-
* to aggregate documents by geographic grids: either [`geo_hash`](/reference/aggregations/search-aggregations-bucket-geohashgrid-aggregation.md), [`geo_tile`](/reference/aggregations/search-aggregations-bucket-geotilegrid-aggregation.md) or [`geo_hex`](/reference/aggregations/search-aggregations-bucket-geohexgrid-aggregation.md).
15-
* to aggregate geopoints into a track using the metrics aggregation [`geo_line`](/reference/aggregations/search-aggregations-metrics-geo-line.md).
16+
* to aggregate documents by geographic grids: either
17+
[`geo_hash`](/reference/aggregations/search-aggregations-bucket-geohashgrid-aggregation.md),
18+
[`geo_tile`](/reference/aggregations/search-aggregations-bucket-geotilegrid-aggregation.md) or
19+
[`geo_hex`](/reference/aggregations/search-aggregations-bucket-geohexgrid-aggregation.md).
20+
* to aggregate geopoints into a track using the metrics aggregation
21+
[`geo_line`](/reference/aggregations/search-aggregations-metrics-geo-line.md).
1622
* to integrate distance into a document’s [relevance score](/reference/query-languages/query-dsl/query-dsl-function-score-query.md).
1723
* to [sort](/reference/elasticsearch/rest-apis/sort-search-results.md#geo-sorting) documents by distance.
1824

19-
As with [geo_shape](/reference/elasticsearch/mapping-reference/geo-shape.md) and [point](/reference/elasticsearch/mapping-reference/point.md), `geo_point` can be specified in [GeoJSON](http://geojson.org) and [Well-Known Text](https://docs.opengeospatial.org/is/12-063r5/12-063r5.html) formats. However, there are a number of additional formats that are supported for convenience and historical reasons. In total there are six ways that a geopoint may be specified, as demonstrated below:
25+
As with [geo_shape](/reference/elasticsearch/mapping-reference/geo-shape.md) and [point](/reference/elasticsearch/mapping-reference/point.md), `geo_point` can be specified in [GeoJSON](http://geojson.org)
26+
and [Well-Known Text](https://docs.opengeospatial.org/is/12-063r5/12-063r5.html) formats.
27+
However, there are a number of additional formats that are supported for convenience and historical reasons.
28+
In total there are six ways that a geopoint may be specified, as demonstrated below:
2029

2130
```console
2231
PUT my-index-000001
@@ -103,15 +112,28 @@ GET my-index-000001/_search
103112
::::{admonition} Geopoints expressed as an array or string
104113
:class: important
105114

106-
Please note that string geopoints are ordered as `lat,lon`, while array geopoints, GeoJSON and WKT are ordered as the reverse: `lon,lat`.
115+
Please note that string geopoints are ordered as `lat,lon`, while array
116+
geopoints, GeoJSON and WKT are ordered as the reverse: `lon,lat`.
107117

108-
The reasons for this are historical. Geographers traditionally write `latitude` before `longitude`, while recent formats specified for geographic data like [GeoJSON](https://geojson.org/) and [Well-Known Text](https://docs.opengeospatial.org/is/12-063r5/12-063r5.html) order `longitude` before `latitude` (easting before northing) in order to match the mathematical convention of ordering `x` before `y`.
118+
The reasons for this are historical. Geographers traditionally write `latitude`
119+
before `longitude`, while recent formats specified for geographic data like
120+
[GeoJSON](https://geojson.org/) and [Well-Known Text](https://docs.opengeospatial.org/is/12-063r5/12-063r5.html)
121+
order `longitude` before `latitude` (easting before northing) in order to match
122+
the mathematical convention of ordering `x` before `y`.
109123

110124
::::
111125

112126

113127
::::{note}
114-
A point can be expressed as a [geohash](https://en.wikipedia.org/wiki/Geohash). Geohashes are [base32](https://en.wikipedia.org/wiki/Base32) encoded strings of the bits of the latitude and longitude interleaved. Each character in a geohash adds additional 5 bits to the precision. So the longer the hash, the more precise it is. For the indexing purposed geohashs are translated into latitude-longitude pairs. During this process only first 12 characters are used, so specifying more than 12 characters in a geohash doesn’t increase the precision. The 12 characters provide 60 bits, which should reduce a possible error to less than 2cm.
128+
A point can be expressed as a [geohash](https://en.wikipedia.org/wiki/Geohash).
129+
Geohashes are [base32](https://en.wikipedia.org/wiki/Base32) encoded strings of
130+
the bits of the latitude and longitude interleaved. Each character in a geohash
131+
adds additional 5 bits to the precision. So the longer the hash, the more
132+
precise it is. For the indexing purposed geohashs are translated into
133+
latitude-longitude pairs. During this process only first 12 characters are
134+
used, so specifying more than 12 characters in a geohash doesn’t increase the
135+
precision. The 12 characters provide 60 bits, which should reduce a possible
136+
error to less than 2cm.
115137
::::
116138

117139

@@ -120,27 +142,54 @@ A point can be expressed as a [geohash](https://en.wikipedia.org/wiki/Geohash).
120142
The following parameters are accepted by `geo_point` fields:
121143

122144
[`ignore_malformed`](/reference/elasticsearch/mapping-reference/ignore-malformed.md)
123-
: If `true`, malformed geopoints are ignored. If `false` (default), malformed geopoints throw an exception and reject the whole document. A geopoint is considered malformed if its latitude is outside the range -90 ⇐ latitude ⇐ 90, or if its longitude is outside the range -180 ⇐ longitude ⇐ 180. Note that this cannot be set if the `script` parameter is used.
145+
: If `true`, malformed geopoints are ignored.
146+
If `false` (default), malformed geopoints throw an exception and reject the whole document.
147+
A geopoint is considered malformed if its latitude is outside the range -90 ⇐ latitude ⇐ 90,
148+
or if its longitude is outside the range -180 ⇐ longitude ⇐ 180.
149+
When set to `true`, if the format is valid, but the values are out of range,
150+
the values will be normalized into the valid range, and the document will be indexed.
151+
This is a special case, and a [different behaviour](/reference/elasticsearch/mapping-reference/ignore-malformed.md#_ignore_malformed_geo_point) from the normal for `ignore_malformed`.
152+
Note that this cannot be set if the `script` parameter is used.
124153

125154
`ignore_z_value`
126-
: If `true` (default) three dimension points will be accepted (stored in source) but only latitude and longitude values will be indexed; the third dimension is ignored. If `false`, geopoints containing any more than latitude and longitude (two dimensions) values throw an exception and reject the whole document. Note that this cannot be set if the `script` parameter is used.
155+
: If `true` (default) three dimension points will be accepted (stored in source)
156+
but only latitude and longitude values will be indexed; the third dimension is
157+
ignored. If `false`, geopoints containing any more than latitude and longitude
158+
(two dimensions) values throw an exception and reject the whole document. Note
159+
that this cannot be set if the `script` parameter is used.
127160

128161
[`index`](/reference/elasticsearch/mapping-reference/mapping-index.md)
129-
: Should the field be quickly searchable? Accepts `true` (default) and `false`. Fields that only have [`doc_values`](/reference/elasticsearch/mapping-reference/doc-values.md) enabled can still be queried, albeit slower.
162+
: Should the field be quickly searchable? Accepts `true` (default) and
163+
`false`. Fields that only have [`doc_values`](/reference/elasticsearch/mapping-reference/doc-values.md)
164+
enabled can still be queried, albeit slower.
130165

131166
[`null_value`](/reference/elasticsearch/mapping-reference/null-value.md)
132-
: Accepts an geopoint value which is substituted for any explicit `null` values. Defaults to `null`, which means the field is treated as missing. Note that this cannot be set if the `script` parameter is used.
167+
: Accepts a geopoint value which is substituted for any explicit `null` values.
168+
Defaults to `null`, which means the field is treated as missing. Note that this
169+
cannot be set if the `script` parameter is used.
133170

134171
`on_script_error`
135-
: Defines what to do if the script defined by the `script` parameter throws an error at indexing time. Accepts `fail` (default), which will cause the entire document to be rejected, and `continue`, which will register the field in the document’s [`_ignored`](/reference/elasticsearch/mapping-reference/mapping-ignored-field.md) metadata field and continue indexing. This parameter can only be set if the `script` field is also set.
172+
: Defines what to do if the script defined by the `script` parameter
173+
throws an error at indexing time. Accepts `fail` (default), which
174+
will cause the entire document to be rejected, and `continue`, which
175+
will register the field in the document’s [`_ignored`](/reference/elasticsearch/mapping-reference/mapping-ignored-field.md) metadata field and continue
176+
indexing. This parameter can only be set if the `script` field is
177+
also set.
136178

137179
`script`
138-
: If this parameter is set, then the field will index values generated by this script, rather than reading the values directly from the source. If a value is set for this field on the input document, then the document will be rejected with an error. Scripts are in the same format as their [runtime equivalent](docs-content://manage-data/data-store/mapping/map-runtime-field.md), and should emit points as a pair of (lat, lon) double values.
180+
: If this parameter is set, then the field will index values generated
181+
by this script, rather than reading the values directly from the
182+
source. If a value is set for this field on the input document, then
183+
the document will be rejected with an error.
184+
Scripts are in the same format as their [runtime equivalent](docs-content://manage-data/data-store/mapping/map-runtime-field.md), and should emit points
185+
as a pair of (lat, lon) double values.
139186

140187

141188
## Using geopoints in scripts [_using_geopoints_in_scripts]
142189

143-
When accessing the value of a geopoint in a script, the value is returned as a `GeoPoint` object, which allows access to the `.lat` and `.lon` values respectively:
190+
When accessing the value of a geopoint in a script, the value is returned as
191+
a `GeoPoint` object, which allows access to the `.lat` and `.lon` values
192+
respectively:
144193

145194
```painless
146195
def geopoint = doc['location'].value;
@@ -159,11 +208,17 @@ def lon = doc['location'].lon;
159208
## Synthetic source [geo-point-synthetic-source]
160209

161210
::::{important}
162-
Synthetic `_source` is Generally Available only for TSDB indices (indices that have `index.mode` set to `time_series`). For other indices synthetic `_source` is in technical preview. Features in technical preview may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features.
211+
Synthetic `_source` is Generally Available only for TSDB indices
212+
(indices that have `index.mode` set to `time_series`). For other indices
213+
synthetic `_source` is in technical preview. Features in technical preview may
214+
be changed or removed in a future release. Elastic will work to fix
215+
any issues, but features in technical preview are not subject to the support SLA
216+
of official GA features.
163217
::::
164218

165219

166-
Synthetic source may sort `geo_point` fields (first by latitude and then longitude) and reduces them to their stored precision. For example:
220+
Synthetic source may sort `geo_point` fields (first by latitude and then
221+
longitude) and reduces them to their stored precision. For example:
167222

168223
$$$synthetic-source-geo-point-example$$$
169224

0 commit comments

Comments
 (0)