Skip to content

Commit daa414e

Browse files
ldematteldematte
committed
Merge remote-tracking branch 'upstream/main' into entitlements/files-policy-base-dir
2 parents 4cf122c + 1bcb68d commit daa414e

File tree

87 files changed

+5750
-2668
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

87 files changed

+5750
-2668
lines changed

‎docs/changelog/119886.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 119886
2+
summary: Initial support for unmapped fields
3+
area: ES|QL
4+
type: feature
5+
issues: []

‎docs/changelog/121370.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 121370
2+
summary: Improve SLM Health Indicator to cover missing snapshot
3+
area: ILM+SLM
4+
type: enhancement
5+
issues: []

‎libs/core/src/main/java/org/elasticsearch/core/TimeValue.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ public class TimeValue implements Comparable<TimeValue> {
2323
public static final TimeValue MAX_VALUE = new TimeValue(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
2424
public static final TimeValue THIRTY_SECONDS = new TimeValue(30, TimeUnit.SECONDS);
2525
public static final TimeValue ONE_MINUTE = new TimeValue(1, TimeUnit.MINUTES);
26+
public static final TimeValue ONE_HOUR = new TimeValue(1, TimeUnit.HOURS);
2627

2728
private static final long C0 = 1L;
2829
private static final long C1 = C0 * 1000L;

‎libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -128,9 +128,7 @@ private static Class<?>[] findClassesToRetransform(Class<?>[] loadedClasses, Set
128128
private static PolicyManager createPolicyManager() {
129129
EntitlementBootstrap.BootstrapArgs bootstrapArgs = EntitlementBootstrap.bootstrapArgs();
130130
Map<String, Policy> pluginPolicies = bootstrapArgs.pluginPolicies();
131-
Path[] dataDirs = EntitlementBootstrap.bootstrapArgs().dataDirs();
132-
133-
var pathLookup = new PathLookup(bootstrapArgs.configDir(), bootstrapArgs.dataDirs(), bootstrapArgs.configDir());
131+
var pathLookup = new PathLookup(bootstrapArgs.configDir(), bootstrapArgs.dataDirs(), bootstrapArgs.tempDir());
134132

135133
// TODO(ES-10031): Decide what goes in the elasticsearch default policy and extend it
136134
var serverPolicy = new Policy(

‎libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121

2222
public final class FileAccessTree {
2323

24-
public static final FileAccessTree EMPTY = new FileAccessTree(FilesEntitlement.EMPTY, null);
2524
private static final String FILE_SEPARATOR = getDefaultFileSystem().getSeparator();
2625

2726
private final String[] readPaths;
@@ -42,6 +41,10 @@ private FileAccessTree(FilesEntitlement filesEntitlement, PathLookup pathLookup)
4241
});
4342
}
4443

44+
// everything has access to the temp dir
45+
readPaths.add(pathLookup.tempDir().toString());
46+
writePaths.add(pathLookup.tempDir().toString());
47+
4548
readPaths.sort(String::compareTo);
4649
writePaths.sort(String::compareTo);
4750

‎libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyManager.java‎

Lines changed: 27 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -70,24 +70,6 @@ record ModuleEntitlements(
7070
entitlementsByType = Map.copyOf(entitlementsByType);
7171
}
7272

73-
public static ModuleEntitlements none(String componentName) {
74-
return new ModuleEntitlements(componentName, Map.of(), FileAccessTree.EMPTY);
75-
}
76-
77-
public static ModuleEntitlements from(String componentName, List<Entitlement> entitlements, PathLookup pathLookup) {
78-
FilesEntitlement filesEntitlement = FilesEntitlement.EMPTY;
79-
for (Entitlement entitlement : entitlements) {
80-
if (entitlement instanceof FilesEntitlement) {
81-
filesEntitlement = (FilesEntitlement) entitlement;
82-
}
83-
}
84-
return new ModuleEntitlements(
85-
componentName,
86-
entitlements.stream().collect(groupingBy(Entitlement::getClass)),
87-
FileAccessTree.of(filesEntitlement, pathLookup)
88-
);
89-
}
90-
9173
public boolean hasEntitlement(Class<? extends Entitlement> entitlementClass) {
9274
return entitlementsByType.containsKey(entitlementClass);
9375
}
@@ -101,13 +83,34 @@ public <E extends Entitlement> Stream<E> getEntitlements(Class<E> entitlementCla
10183
}
10284
}
10385

86+
// pkg private for testing
87+
ModuleEntitlements defaultEntitlements(String componentName) {
88+
return new ModuleEntitlements(componentName, Map.of(), defaultFileAccess);
89+
}
90+
91+
// pkg private for testing
92+
ModuleEntitlements policyEntitlements(String componentName, List<Entitlement> entitlements) {
93+
FilesEntitlement filesEntitlement = FilesEntitlement.EMPTY;
94+
for (Entitlement entitlement : entitlements) {
95+
if (entitlement instanceof FilesEntitlement) {
96+
filesEntitlement = (FilesEntitlement) entitlement;
97+
}
98+
}
99+
return new ModuleEntitlements(
100+
componentName,
101+
entitlements.stream().collect(groupingBy(Entitlement::getClass)),
102+
FileAccessTree.of(filesEntitlement, pathLookup)
103+
);
104+
}
105+
104106
final Map<Module, ModuleEntitlements> moduleEntitlementsMap = new ConcurrentHashMap<>();
105107

106108
private final Map<String, List<Entitlement>> serverEntitlements;
107109
private final List<Entitlement> apmAgentEntitlements;
108110
private final Map<String, Map<String, List<Entitlement>>> pluginsEntitlements;
109111
private final Function<Class<?>, String> pluginResolver;
110112
private final PathLookup pathLookup;
113+
private final FileAccessTree defaultFileAccess;
111114

112115
public static final String ALL_UNNAMED = "ALL-UNNAMED";
113116

@@ -154,6 +157,7 @@ public PolicyManager(
154157
this.apmAgentPackageName = apmAgentPackageName;
155158
this.entitlementsModule = entitlementsModule;
156159
this.pathLookup = requireNonNull(pathLookup);
160+
this.defaultFileAccess = FileAccessTree.of(FilesEntitlement.EMPTY, pathLookup);
157161

158162
for (var e : serverEntitlements.entrySet()) {
159163
validateEntitlementsPerModule(SERVER_COMPONENT_NAME, e.getKey(), e.getValue());
@@ -428,7 +432,7 @@ private ModuleEntitlements computeEntitlements(Class<?> requestingClass) {
428432
if (pluginName != null) {
429433
var pluginEntitlements = pluginsEntitlements.get(pluginName);
430434
if (pluginEntitlements == null) {
431-
return ModuleEntitlements.none(pluginName);
435+
return defaultEntitlements(pluginName);
432436
} else {
433437
final String scopeName;
434438
if (requestingModule.isNamed() == false) {
@@ -442,10 +446,10 @@ private ModuleEntitlements computeEntitlements(Class<?> requestingClass) {
442446

443447
if (requestingModule.isNamed() == false && requestingClass.getPackageName().startsWith(apmAgentPackageName)) {
444448
// The APM agent is the only thing running non-modular in the system classloader
445-
return ModuleEntitlements.from(APM_AGENT_COMPONENT_NAME, apmAgentEntitlements, pathLookup);
449+
return policyEntitlements(APM_AGENT_COMPONENT_NAME, apmAgentEntitlements);
446450
}
447451

448-
return ModuleEntitlements.none(UNKNOWN_COMPONENT_NAME);
452+
return defaultEntitlements(UNKNOWN_COMPONENT_NAME);
449453
}
450454

451455
private ModuleEntitlements getModuleScopeEntitlements(
@@ -455,9 +459,9 @@ private ModuleEntitlements getModuleScopeEntitlements(
455459
) {
456460
var entitlements = scopeEntitlements.get(moduleName);
457461
if (entitlements == null) {
458-
return ModuleEntitlements.none(componentName);
462+
return defaultEntitlements(componentName);
459463
}
460-
return ModuleEntitlements.from(componentName, entitlements, pathLookup);
464+
return policyEntitlements(componentName, entitlements);
461465
}
462466

463467
private static boolean isServerModule(Module requestingModule) {

‎libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTreeTests.java‎

Lines changed: 29 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -35,18 +35,20 @@ private static Path path(String s) {
3535
return root.resolve(s);
3636
}
3737

38-
private static PathLookup createTestPathLookup() {
39-
return new PathLookup(Path.of("/config"), new Path[] { Path.of("/data1"), Path.of("/data2") }, Path.of("/tmp"));
40-
}
38+
private static final PathLookup TEST_PATH_LOOKUP = new PathLookup(
39+
Path.of("/config"),
40+
new Path[] { Path.of("/data1"), Path.of("/data2") },
41+
Path.of("/tmp")
42+
);
4143

4244
public void testEmpty() {
43-
var tree = FileAccessTree.of(FilesEntitlement.EMPTY, createTestPathLookup());
45+
var tree = accessTree(FilesEntitlement.EMPTY);
4446
assertThat(tree.canRead(path("path")), is(false));
4547
assertThat(tree.canWrite(path("path")), is(false));
4648
}
4749

4850
public void testRead() {
49-
var tree = FileAccessTree.of(entitlement("foo", "read"), createTestPathLookup());
51+
var tree = accessTree(entitlement("foo", "read"));
5052
assertThat(tree.canRead(path("foo")), is(true));
5153
assertThat(tree.canRead(path("foo/subdir")), is(true));
5254
assertThat(tree.canRead(path("food")), is(false));
@@ -58,7 +60,7 @@ public void testRead() {
5860
}
5961

6062
public void testWrite() {
61-
var tree = FileAccessTree.of(entitlement("foo", "read_write"), createTestPathLookup());
63+
var tree = accessTree(entitlement("foo", "read_write"));
6264
assertThat(tree.canWrite(path("foo")), is(true));
6365
assertThat(tree.canWrite(path("foo/subdir")), is(true));
6466
assertThat(tree.canWrite(path("food")), is(false));
@@ -70,7 +72,7 @@ public void testWrite() {
7072
}
7173

7274
public void testTwoPaths() {
73-
var tree = FileAccessTree.of(entitlement("foo", "read", "bar", "read"), createTestPathLookup());
75+
var tree = accessTree(entitlement("foo", "read", "bar", "read"));
7476
assertThat(tree.canRead(path("a")), is(false));
7577
assertThat(tree.canRead(path("bar")), is(true));
7678
assertThat(tree.canRead(path("bar/subdir")), is(true));
@@ -81,16 +83,15 @@ public void testTwoPaths() {
8183
}
8284

8385
public void testReadWriteUnderRead() {
84-
var tree = FileAccessTree.of(entitlement("foo", "read", "foo/bar", "read_write"), createTestPathLookup());
86+
var tree = accessTree(entitlement("foo", "read", "foo/bar", "read_write"));
8587
assertThat(tree.canRead(path("foo")), is(true));
8688
assertThat(tree.canWrite(path("foo")), is(false));
8789
assertThat(tree.canRead(path("foo/bar")), is(true));
8890
assertThat(tree.canWrite(path("foo/bar")), is(true));
8991
}
9092

9193
public void testReadWithRelativePath() {
92-
var resolver = createTestPathLookup();
93-
var tree = FileAccessTree.of(entitlement(Map.of("relative_path", "foo", "mode", "read", "relative_to", "config")), resolver);
94+
var tree = accessTree(entitlement(Map.of("relative_path", "foo", "mode", "read", "relative_to", "config")));
9495
assertThat(tree.canRead(path("foo")), is(false));
9596

9697
assertThat(tree.canRead(path("/config/foo")), is(true));
@@ -105,8 +106,7 @@ public void testReadWithRelativePath() {
105106
}
106107

107108
public void testWriteWithRelativePath() {
108-
var resolver = createTestPathLookup();
109-
var tree = FileAccessTree.of(entitlement(Map.of("relative_path", "foo", "mode", "read_write", "relative_to", "config")), resolver);
109+
var tree = accessTree(entitlement(Map.of("relative_path", "foo", "mode", "read_write", "relative_to", "config")));
110110
assertThat(tree.canWrite(path("/config/foo")), is(true));
111111
assertThat(tree.canWrite(path("/config/foo/subdir")), is(true));
112112
assertThat(tree.canWrite(path("foo")), is(false));
@@ -120,8 +120,7 @@ public void testWriteWithRelativePath() {
120120
}
121121

122122
public void testMultipleDataDirs() {
123-
var resolver = createTestPathLookup();
124-
var tree = FileAccessTree.of(entitlement(Map.of("relative_path", "foo", "mode", "read_write", "relative_to", "data")), resolver);
123+
var tree = accessTree(entitlement(Map.of("relative_path", "foo", "mode", "read_write", "relative_to", "data")));
125124
assertThat(tree.canWrite(path("/data1/foo")), is(true));
126125
assertThat(tree.canWrite(path("/data2/foo")), is(true));
127126
assertThat(tree.canWrite(path("/data3/foo")), is(false));
@@ -139,15 +138,15 @@ public void testMultipleDataDirs() {
139138
}
140139

141140
public void testNormalizePath() {
142-
var tree = FileAccessTree.of(entitlement("foo/../bar", "read"), createTestPathLookup());
141+
var tree = accessTree(entitlement("foo/../bar", "read"));
143142
assertThat(tree.canRead(path("foo/../bar")), is(true));
144143
assertThat(tree.canRead(path("foo")), is(false));
145144
assertThat(tree.canRead(path("")), is(false));
146145
}
147146

148147
public void testForwardSlashes() {
149148
String sep = getDefaultFileSystem().getSeparator();
150-
var tree = FileAccessTree.of(entitlement("a/b", "read", "m" + sep + "n", "read"), createTestPathLookup());
149+
var tree = accessTree(entitlement("a/b", "read", "m" + sep + "n", "read"));
151150

152151
// Native separators work
153152
assertThat(tree.canRead(path("a" + sep + "b")), is(true));
@@ -158,6 +157,20 @@ public void testForwardSlashes() {
158157
assertThat(tree.canRead(path("m/n")), is(true));
159158
}
160159

160+
public void testTempDirAccess() {
161+
Path tempDir = createTempDir();
162+
var tree = FileAccessTree.of(
163+
FilesEntitlement.EMPTY,
164+
new PathLookup(Path.of("/config"), new Path[] { Path.of("/data1"), Path.of("/data2") }, tempDir)
165+
);
166+
assertThat(tree.canRead(tempDir), is(true));
167+
assertThat(tree.canWrite(tempDir), is(true));
168+
}
169+
170+
FileAccessTree accessTree(FilesEntitlement entitlement) {
171+
return FileAccessTree.of(entitlement, TEST_PATH_LOOKUP);
172+
}
173+
161174
static FilesEntitlement entitlement(String... values) {
162175
List<Object> filesData = new ArrayList<>();
163176
for (int i = 0; i < values.length; i += 2) {

‎libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -84,9 +84,13 @@ public void testGetEntitlementsThrowsOnMissingPluginUnnamedModule() {
8484
var callerClass = this.getClass();
8585
var requestingModule = callerClass.getModule();
8686

87-
assertEquals("No policy for the unnamed module", ModuleEntitlements.none("plugin1"), policyManager.getEntitlements(callerClass));
87+
assertEquals(
88+
"No policy for the unnamed module",
89+
policyManager.defaultEntitlements("plugin1"),
90+
policyManager.getEntitlements(callerClass)
91+
);
8892

89-
assertEquals(Map.of(requestingModule, ModuleEntitlements.none("plugin1")), policyManager.moduleEntitlementsMap);
93+
assertEquals(Map.of(requestingModule, policyManager.defaultEntitlements("plugin1")), policyManager.moduleEntitlementsMap);
9094
}
9195

9296
public void testGetEntitlementsThrowsOnMissingPolicyForPlugin() {
@@ -104,9 +108,9 @@ public void testGetEntitlementsThrowsOnMissingPolicyForPlugin() {
104108
var callerClass = this.getClass();
105109
var requestingModule = callerClass.getModule();
106110

107-
assertEquals("No policy for this plugin", ModuleEntitlements.none("plugin1"), policyManager.getEntitlements(callerClass));
111+
assertEquals("No policy for this plugin", policyManager.defaultEntitlements("plugin1"), policyManager.getEntitlements(callerClass));
108112

109-
assertEquals(Map.of(requestingModule, ModuleEntitlements.none("plugin1")), policyManager.moduleEntitlementsMap);
113+
assertEquals(Map.of(requestingModule, policyManager.defaultEntitlements("plugin1")), policyManager.moduleEntitlementsMap);
110114
}
111115

112116
public void testGetEntitlementsFailureIsCached() {
@@ -124,14 +128,14 @@ public void testGetEntitlementsFailureIsCached() {
124128
var callerClass = this.getClass();
125129
var requestingModule = callerClass.getModule();
126130

127-
assertEquals(ModuleEntitlements.none("plugin1"), policyManager.getEntitlements(callerClass));
128-
assertEquals(Map.of(requestingModule, ModuleEntitlements.none("plugin1")), policyManager.moduleEntitlementsMap);
131+
assertEquals(policyManager.defaultEntitlements("plugin1"), policyManager.getEntitlements(callerClass));
132+
assertEquals(Map.of(requestingModule, policyManager.defaultEntitlements("plugin1")), policyManager.moduleEntitlementsMap);
129133

130134
// A second time
131-
assertEquals(ModuleEntitlements.none("plugin1"), policyManager.getEntitlements(callerClass));
135+
assertEquals(policyManager.defaultEntitlements("plugin1"), policyManager.getEntitlements(callerClass));
132136

133137
// Nothing new in the map
134-
assertEquals(Map.of(requestingModule, ModuleEntitlements.none("plugin1")), policyManager.moduleEntitlementsMap);
138+
assertEquals(Map.of(requestingModule, policyManager.defaultEntitlements("plugin1")), policyManager.moduleEntitlementsMap);
135139
}
136140

137141
public void testGetEntitlementsReturnsEntitlementsForPluginUnnamedModule() {
@@ -172,11 +176,14 @@ public void testGetEntitlementsThrowsOnMissingPolicyForServer() throws ClassNotF
172176

173177
assertEquals(
174178
"No policy for this module in server",
175-
ModuleEntitlements.none(SERVER_COMPONENT_NAME),
179+
policyManager.defaultEntitlements(SERVER_COMPONENT_NAME),
176180
policyManager.getEntitlements(mockServerClass)
177181
);
178182

179-
assertEquals(Map.of(requestingModule, ModuleEntitlements.none(SERVER_COMPONENT_NAME)), policyManager.moduleEntitlementsMap);
183+
assertEquals(
184+
Map.of(requestingModule, policyManager.defaultEntitlements(SERVER_COMPONENT_NAME)),
185+
policyManager.moduleEntitlementsMap
186+
);
180187
}
181188

182189
public void testGetEntitlementsReturnsEntitlementsForServerModule() throws ClassNotFoundException {

‎modules/ingest-geoip/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/ingest/geoip/FullClusterRestartIT.java‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.elasticsearch.common.settings.Settings;
2121
import org.elasticsearch.common.util.concurrent.ThreadContext;
2222
import org.elasticsearch.core.Nullable;
23+
import org.elasticsearch.rest.RestStatus;
2324
import org.elasticsearch.test.cluster.ElasticsearchCluster;
2425
import org.elasticsearch.test.cluster.FeatureFlag;
2526
import org.elasticsearch.test.cluster.local.distribution.DistributionType;
@@ -170,7 +171,7 @@ public void testGeoIpSystemFeaturesMigration() throws Exception {
170171
@SuppressWarnings("unchecked")
171172
private void testDatabasesLoaded() throws IOException {
172173
Request getTaskState = new Request("GET", "/_cluster/state");
173-
ObjectPath state = ObjectPath.createFromResponse(client().performRequest(getTaskState));
174+
ObjectPath state = ObjectPath.createFromResponse(assertOK(client().performRequest(getTaskState)));
174175

175176
List<?> tasks = state.evaluate("metadata.persistent_tasks.tasks");
176177
// Short-circuit to avoid using steams if the list is empty
@@ -196,7 +197,10 @@ private void testDatabasesLoaded() throws IOException {
196197

197198
private void testCatIndices(List<String> indexNames, @Nullable List<String> additionalIndexNames) throws IOException {
198199
Request catIndices = new Request("GET", "_cat/indices/*?s=index&h=index&expand_wildcards=all");
199-
String response = EntityUtils.toString(client().performRequest(catIndices).getEntity());
200+
// the cat APIs can sometimes 404, erroneously
201+
// see https://github.com/elastic/elasticsearch/issues/104371
202+
setIgnoredErrorResponseCodes(catIndices, RestStatus.NOT_FOUND);
203+
String response = EntityUtils.toString(assertOK(client().performRequest(catIndices)).getEntity());
200204
List<String> indices = List.of(response.trim().split("\\s+"));
201205

202206
if (additionalIndexNames != null && additionalIndexNames.isEmpty() == false) {
@@ -215,7 +219,7 @@ private void testIndexGeoDoc() throws IOException {
215219
assertOK(client().performRequest(putDoc));
216220

217221
Request getDoc = new Request("GET", "/my-index-00001/_doc/my_id");
218-
ObjectPath doc = ObjectPath.createFromResponse(client().performRequest(getDoc));
222+
ObjectPath doc = ObjectPath.createFromResponse(assertOK(client().performRequest(getDoc)));
219223
assertNull(doc.evaluate("_source.tags"));
220224
assertEquals("Sweden", doc.evaluate("_source.geo.country_name"));
221225
}
@@ -225,8 +229,7 @@ private void testGetStar(List<String> indexNames, @Nullable List<String> additio
225229
getStar.setOptions(
226230
RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE) // we don't care about warnings, just errors
227231
);
228-
Response response = client().performRequest(getStar);
229-
assertOK(response);
232+
Response response = assertOK(client().performRequest(getStar));
230233

231234
if (additionalIndexNames != null && additionalIndexNames.isEmpty() == false) {
232235
indexNames = new ArrayList<>(indexNames); // recopy into a mutable list
@@ -244,8 +247,7 @@ private void testGetStarAsKibana(List<String> indexNames, @Nullable List<String>
244247
.addHeader("X-elastic-product-origin", "kibana")
245248
.setWarningsHandler(WarningsHandler.PERMISSIVE) // we don't care about warnings, just errors
246249
);
247-
Response response = client().performRequest(getStar);
248-
assertOK(response);
250+
Response response = assertOK(client().performRequest(getStar));
249251

250252
if (additionalIndexNames != null && additionalIndexNames.isEmpty() == false) {
251253
indexNames = new ArrayList<>(indexNames); // recopy into a mutable list

0 commit comments

Comments
 (0)