Skip to content

Commit 63fc87a

Browse files
DaveCTurnerDaveCTurner
committed
Merge branch 'main' into 2022-12-14-blobstore-metadata-integrity-checks
2 parents 7a88397 + d7cc19c commit 63fc87a

File tree

32 files changed

+1115
-317
lines changed

32 files changed

+1115
-317
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/vector/DistanceFunctionBenchmark.java‎

Lines changed: 467 additions & 0 deletions
Large diffs are not rendered by default.

‎docs/changelog/92314.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 92314
2+
summary: JWT realm - add support for required claims
3+
area: Authentication
4+
type: enhancement
5+
issues: []

‎docs/changelog/92340.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 92340
2+
summary: Add vector distance scoring to micro benchmarks
3+
area: Performance
4+
type: enhancement
5+
issues: []

‎docs/changelog/92372.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
pr: 92372
2+
summary: Speed up ingest geoip processors
3+
area: Ingest Node
4+
type: bug
5+
issues: []
6+
highlight:
7+
title: Speed up ingest geoip processors
8+
body: |-
9+
The `geoip` ingest processor is significantly faster.
10+
11+
Previous versions of the geoip library needed special permission to execute
12+
databinding code, requiring an expensive permissions check and
13+
`AccessController.doPrivileged` call. The current version of the geoip
14+
library no longer requires that, however, so the expensive code has been
15+
removed, resulting in better performance for the ingest geoip processor.
16+
notable: true

‎docs/reference/index-custom-title-page.html‎

Lines changed: 37 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -68,44 +68,6 @@ <h2>Search and analyze your data</h2>
6868
</div>
6969
</div>
7070

71-
<h3>Explore by use case</h3>
72-
73-
<div class="row my-4">
74-
<div class="col-md-4 col-12 mb-2">
75-
<a class="no-text-decoration" href="https://www.elastic.co/guide/en/enterprise-search/current/start.html">
76-
<div class="card h-100">
77-
<h4 class="mt-3">
78-
<span class="inline-block float-left icon mr-2" style="background-image: url('https://images.contentstack.io/v3/assets/bltefdd0b53724fa2ce/blt11200907c1c033aa/634d9da119d8652169cf9b2b/enterprise-search-logo-color-32px.png');"></span>
79-
Search my data
80-
</h4>
81-
<p>Create search experiences for your content, wherever it lives.</p>
82-
</div>
83-
</a>
84-
</div>
85-
<div class="col-md-4 col-12 mb-2">
86-
<a class="no-text-decoration" href="https://www.elastic.co/guide/en/welcome-to-elastic/current/getting-started-observability.html">
87-
<div class="card h-100">
88-
<h4 class="mt-3">
89-
<span class="inline-block float-left icon mr-2" style="background-image: url('https://images.contentstack.io/v3/assets/bltefdd0b53724fa2ce/bltaa08b370a00bbecc/634d9da14e565f1cdce27f7c/observability-logo-color-32px.png');"></span>
90-
Observe my data
91-
</h4>
92-
<p>Follow our guides to monitor logs, metrics, and traces.</p>
93-
</div>
94-
</a>
95-
</div>
96-
<div class="col-md-4 col-12 mb-2">
97-
<a class="no-text-decoration" href="https://www.elastic.co/guide/en/security/current/es-overview.html">
98-
<div class="card h-100">
99-
<h4 class="mt-3">
100-
<span class="inline-block float-left icon mr-2" style="background-image: url('https://images.contentstack.io/v3/assets/bltefdd0b53724fa2ce/blt5e0e0ad9a13e6b8c/634d9da18473831f96bbdf1e/security-logo-color-32px.png');"></span>
101-
Protect my environment
102-
</h4>
103-
<p>Learn how to defend against threats across your environment.</p>
104-
</div>
105-
</a>
106-
</div>
107-
</div>
108-
10971
<h3>Get to know Elasticsearch</h3>
11072

11173
<div class="my-5">
@@ -229,5 +191,42 @@ <h4 class="mt-3">
229191
</ul>
230192
</div>
231193

194+
<h3>Explore by use case</h3>
195+
196+
<div class="row my-4">
197+
<div class="col-md-4 col-12 mb-2">
198+
<a class="no-text-decoration" href="https://www.elastic.co/guide/en/enterprise-search/current/start.html">
199+
<div class="card h-100">
200+
<h4 class="mt-3">
201+
<span class="inline-block float-left icon mr-2" style="background-image: url('https://images.contentstack.io/v3/assets/bltefdd0b53724fa2ce/blt11200907c1c033aa/634d9da119d8652169cf9b2b/enterprise-search-logo-color-32px.png');"></span>
202+
Search my data
203+
</h4>
204+
<p>Create search experiences for your content, wherever it lives.</p>
205+
</div>
206+
</a>
207+
</div>
208+
<div class="col-md-4 col-12 mb-2">
209+
<a class="no-text-decoration" href="https://www.elastic.co/guide/en/welcome-to-elastic/current/getting-started-observability.html">
210+
<div class="card h-100">
211+
<h4 class="mt-3">
212+
<span class="inline-block float-left icon mr-2" style="background-image: url('https://images.contentstack.io/v3/assets/bltefdd0b53724fa2ce/bltaa08b370a00bbecc/634d9da14e565f1cdce27f7c/observability-logo-color-32px.png');"></span>
213+
Observe my data
214+
</h4>
215+
<p>Follow our guides to monitor logs, metrics, and traces.</p>
216+
</div>
217+
</a>
218+
</div>
219+
<div class="col-md-4 col-12 mb-2">
220+
<a class="no-text-decoration" href="https://www.elastic.co/guide/en/security/current/es-overview.html">
221+
<div class="card h-100">
222+
<h4 class="mt-3">
223+
<span class="inline-block float-left icon mr-2" style="background-image: url('https://images.contentstack.io/v3/assets/bltefdd0b53724fa2ce/blt5e0e0ad9a13e6b8c/634d9da18473831f96bbdf1e/security-logo-color-32px.png');"></span>
224+
Protect my environment
225+
</h4>
226+
<p>Learn how to defend against threats across your environment.</p>
227+
</div>
228+
</a>
229+
</div>
230+
</div>
232231

233232
<p class="my-4"><a href="https://www.elastic.co/guide/index.html">View all Elastic docs</a></p>

‎libs/h3/src/test/java/org/elasticsearch/h3/AzimuthTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ public void testLatLonVec3d() {
3030
for (int i = 0; i < Vec3d.faceCenterPoint.length; i++) {
3131
final double azVec3d = Vec3d.faceCenterPoint[i].geoAzimuthRads(point.x, point.y, point.z);
3232
final double azVec2d = Vec2d.faceCenterGeo[i].geoAzimuthRads(point.getLatitude(), point.getLongitude());
33-
assertEquals("Face " + i, azVec2d, azVec3d, 1e-14);
33+
assertEquals("Face " + i, azVec2d, azVec3d, 1e-12);
3434
}
3535
}
3636

‎modules/apm/src/main/java/org/elasticsearch/tracing/apm/APM.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ public List<Setting<?>> getSettings() {
101101
APMAgentSettings.APM_ENABLED_SETTING,
102102
APMAgentSettings.APM_TRACING_NAMES_INCLUDE_SETTING,
103103
APMAgentSettings.APM_TRACING_NAMES_EXCLUDE_SETTING,
104+
APMAgentSettings.APM_TRACING_SANITIZE_FIELD_NAMES,
104105
APMAgentSettings.APM_AGENT_SETTINGS,
105106
APMAgentSettings.APM_SECRET_TOKEN_SETTING,
106107
APMAgentSettings.APM_API_KEY_SETTING

‎modules/apm/src/main/java/org/elasticsearch/tracing/apm/APMAgentSettings.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ void addClusterSettingsListeners(ClusterService clusterService, APMTracer apmTra
5757
});
5858
clusterSettings.addSettingsUpdateConsumer(APM_TRACING_NAMES_INCLUDE_SETTING, apmTracer::setIncludeNames);
5959
clusterSettings.addSettingsUpdateConsumer(APM_TRACING_NAMES_EXCLUDE_SETTING, apmTracer::setExcludeNames);
60+
clusterSettings.addSettingsUpdateConsumer(APM_TRACING_SANITIZE_FIELD_NAMES, apmTracer::setLabelFilters);
6061
clusterSettings.addAffixMapUpdateConsumer(APM_AGENT_SETTINGS, map -> map.forEach(this::setAgentSetting), (x, y) -> {});
6162
}
6263

@@ -143,6 +144,27 @@ void setAgentSetting(String key, String value) {
143144
NodeScope
144145
);
145146

147+
static final Setting<List<String>> APM_TRACING_SANITIZE_FIELD_NAMES = Setting.listSetting(
148+
APM_SETTING_PREFIX + "sanitize_field_names",
149+
List.of(
150+
"password",
151+
"passwd",
152+
"pwd",
153+
"secret",
154+
"*key",
155+
"*token*",
156+
"*session*",
157+
"*credit*",
158+
"*card*",
159+
"*auth*",
160+
"*principal*",
161+
"set-cookie"
162+
),
163+
Function.identity(),
164+
OperatorDynamic,
165+
NodeScope
166+
);
167+
146168
static final Setting<Boolean> APM_ENABLED_SETTING = Setting.boolSetting(
147169
APM_SETTING_PREFIX + "enabled",
148170
false,

‎modules/apm/src/main/java/org/elasticsearch/tracing/apm/APMTracer.java‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
import static org.elasticsearch.tracing.apm.APMAgentSettings.APM_ENABLED_SETTING;
4545
import static org.elasticsearch.tracing.apm.APMAgentSettings.APM_TRACING_NAMES_EXCLUDE_SETTING;
4646
import static org.elasticsearch.tracing.apm.APMAgentSettings.APM_TRACING_NAMES_INCLUDE_SETTING;
47+
import static org.elasticsearch.tracing.apm.APMAgentSettings.APM_TRACING_SANITIZE_FIELD_NAMES;
4748

4849
/**
4950
* This is an implementation of the {@link org.elasticsearch.tracing.Tracer} interface, which uses
@@ -65,8 +66,10 @@ public class APMTracer extends AbstractLifecycleComponent implements org.elastic
6566

6667
private List<String> includeNames;
6768
private List<String> excludeNames;
69+
private List<String> labelFilters;
6870
/** Built using {@link #includeNames} and {@link #excludeNames}, and filters out spans based on their name. */
6971
private volatile CharacterRunAutomaton filterAutomaton;
72+
private volatile CharacterRunAutomaton labelFilterAutomaton;
7073
private String clusterName;
7174
private String nodeName;
7275

@@ -86,7 +89,10 @@ record APMServices(Tracer tracer, OpenTelemetry openTelemetry) {}
8689
public APMTracer(Settings settings) {
8790
this.includeNames = APM_TRACING_NAMES_INCLUDE_SETTING.get(settings);
8891
this.excludeNames = APM_TRACING_NAMES_EXCLUDE_SETTING.get(settings);
92+
this.labelFilters = APM_TRACING_SANITIZE_FIELD_NAMES.get(settings);
93+
8994
this.filterAutomaton = buildAutomaton(includeNames, excludeNames);
95+
this.labelFilterAutomaton = buildAutomaton(labelFilters, List.of());
9096
this.enabled = APM_ENABLED_SETTING.get(settings);
9197
}
9298

@@ -109,6 +115,16 @@ void setExcludeNames(List<String> excludeNames) {
109115
this.filterAutomaton = buildAutomaton(includeNames, excludeNames);
110116
}
111117

118+
void setLabelFilters(List<String> labelFilters) {
119+
this.labelFilters = labelFilters;
120+
this.labelFilterAutomaton = buildAutomaton(labelFilters, List.of());
121+
}
122+
123+
// package-private for testing
124+
CharacterRunAutomaton getLabelFilterAutomaton() {
125+
return labelFilterAutomaton;
126+
}
127+
112128
@Override
113129
protected void doStart() {
114130
if (enabled) {
@@ -271,6 +287,12 @@ private void setSpanAttributes(@Nullable Map<String, Object> spanAttributes, Spa
271287
for (Map.Entry<String, Object> entry : spanAttributes.entrySet()) {
272288
final String key = entry.getKey();
273289
final Object value = entry.getValue();
290+
291+
if (this.labelFilterAutomaton.run(key)) {
292+
spanBuilder.setAttribute(key, "[REDACTED]");
293+
continue;
294+
}
295+
274296
if (value instanceof String) {
275297
spanBuilder.setAttribute(key, (String) value);
276298
} else if (value instanceof Long) {
@@ -394,9 +416,9 @@ Map<String, Context> getSpans() {
394416
return spans;
395417
}
396418

397-
private static CharacterRunAutomaton buildAutomaton(List<String> includeNames, List<String> excludeNames) {
398-
Automaton includeAutomaton = patternsToAutomaton(includeNames);
399-
Automaton excludeAutomaton = patternsToAutomaton(excludeNames);
419+
private static CharacterRunAutomaton buildAutomaton(List<String> includePatterns, List<String> excludePatterns) {
420+
Automaton includeAutomaton = patternsToAutomaton(includePatterns);
421+
Automaton excludeAutomaton = patternsToAutomaton(excludePatterns);
400422

401423
if (includeAutomaton == null) {
402424
includeAutomaton = Automata.makeAnyString();

‎modules/apm/src/test/java/org/elasticsearch/tracing/apm/APMTracerTests.java‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,14 @@
88

99
package org.elasticsearch.tracing.apm;
1010

11+
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
1112
import org.elasticsearch.common.settings.Settings;
1213
import org.elasticsearch.common.util.concurrent.ThreadContext;
1314
import org.elasticsearch.tasks.Task;
1415
import org.elasticsearch.test.ESTestCase;
1516

1617
import java.util.List;
18+
import java.util.stream.Stream;
1719

1820
import static org.elasticsearch.tracing.apm.APMAgentSettings.APM_ENABLED_SETTING;
1921
import static org.elasticsearch.tracing.apm.APMAgentSettings.APM_TRACING_NAMES_EXCLUDE_SETTING;
@@ -166,6 +168,43 @@ public void test_whenTraceStarted_andSpanNameExcluded_thenSpanIsNotStarted() {
166168
assertThat(apmTracer.getSpans(), hasKey("id3"));
167169
}
168170

171+
/**
172+
* Check that sensitive attributes are not added verbatim to a span, but instead the value is redacted.
173+
*/
174+
public void test_whenAddingAttributes_thenSensitiveValuesAreRedacted() {
175+
Settings settings = Settings.builder().put(APM_ENABLED_SETTING.getKey(), false).build();
176+
APMTracer apmTracer = buildTracer(settings);
177+
CharacterRunAutomaton labelFilterAutomaton = apmTracer.getLabelFilterAutomaton();
178+
179+
Stream.of(
180+
"auth",
181+
"auth-header",
182+
"authValue",
183+
"card",
184+
"card-details",
185+
"card-number",
186+
"credit",
187+
"credit-card",
188+
"key",
189+
"my-credit-number",
190+
"my_session_id",
191+
"passwd",
192+
"password",
193+
"principal",
194+
"principal-value",
195+
"pwd",
196+
"secret",
197+
"secure-key",
198+
"sensitive-token*",
199+
"session",
200+
"session_id",
201+
"set-cookie",
202+
"some-auth",
203+
"some-principal",
204+
"token-for login"
205+
).forEach(key -> assertTrue("Expected label filter automaton to redact [" + key + "]", labelFilterAutomaton.run(key)));
206+
}
207+
169208
private APMTracer buildTracer(Settings settings) {
170209
APMTracer tracer = new APMTracer(settings);
171210
tracer.doStart();

0 commit comments

Comments
 (0)