Skip to content

Commit 26d5823

Browse files
slobodanadamovicslobodanadamovic
authored
Merge branch '9.0' into backport/9.0/pr-121971
2 parents 66447f5 + 777d5d0 commit 26d5823

File tree

72 files changed

+4102
-3977
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

72 files changed

+4102
-3977
lines changed

‎docs/changelog/119886.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 119886
2+
summary: Initial support for unmapped fields
3+
area: ES|QL
4+
type: feature
5+
issues: []

‎libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlement.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,12 +47,10 @@ public sealed interface FileData {
4747
Mode mode();
4848

4949
static FileData ofPath(Path path, Mode mode) {
50-
assert path.isAbsolute();
5150
return new AbsolutePathFileData(path, mode);
5251
}
5352

5453
static FileData ofRelativePath(Path relativePath, BaseDir baseDir, Mode mode) {
55-
assert relativePath.isAbsolute() == false;
5654
return new RelativePathFileData(relativePath, baseDir, mode);
5755
}
5856
}

‎libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyManagerTests.java‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -53,18 +53,23 @@ public class PolicyManagerTests extends ESTestCase {
5353
*/
5454
private static Module NO_ENTITLEMENTS_MODULE;
5555

56-
private static final PathLookup TEST_PATH_LOOKUP = new PathLookup(
57-
Path.of("/user/home"),
58-
Path.of("/config"),
59-
new Path[] { Path.of("/data1/"), Path.of("/data2") },
60-
Path.of("/temp")
61-
);
56+
private static Path TEST_BASE_DIR;
57+
58+
private static PathLookup TEST_PATH_LOOKUP;
6259

6360
@BeforeClass
6461
public static void beforeClass() {
6562
try {
6663
// Any old module will do for tests using NO_ENTITLEMENTS_MODULE
6764
NO_ENTITLEMENTS_MODULE = makeClassInItsOwnModule().getModule();
65+
66+
TEST_BASE_DIR = createTempDir().toAbsolutePath();
67+
TEST_PATH_LOOKUP = new PathLookup(
68+
TEST_BASE_DIR.resolve("/user/home"),
69+
TEST_BASE_DIR.resolve("/config"),
70+
new Path[] { TEST_BASE_DIR.resolve("/data1/"), TEST_BASE_DIR.resolve("/data2") },
71+
TEST_BASE_DIR.resolve("/temp")
72+
);
6873
} catch (Exception e) {
6974
throw new IllegalStateException(e);
7075
}
@@ -229,8 +234,7 @@ public void testGetEntitlementsReturnsEntitlementsForPluginModule() throws IOExc
229234

230235
var entitlements = policyManager.getEntitlements(mockPluginClass);
231236
assertThat(entitlements.hasEntitlement(CreateClassLoaderEntitlement.class), is(true));
232-
// TODO: this can't work on Windows, we need to have the root be unknown
233-
// assertThat(entitlements.fileAccess().canRead("/test/path"), is(true));
237+
assertThat(entitlements.fileAccess().canRead(TEST_BASE_DIR), is(true));
234238
}
235239

236240
public void testGetEntitlementsResultIsCached() {
@@ -440,9 +444,7 @@ private static Policy createPluginPolicy(String... pluginModules) {
440444
name -> new Scope(
441445
name,
442446
List.of(
443-
new FilesEntitlement(
444-
List.of(FilesEntitlement.FileData.ofPath(Path.of("/test/path"), FilesEntitlement.Mode.READ))
445-
),
447+
new FilesEntitlement(List.of(FilesEntitlement.FileData.ofPath(TEST_BASE_DIR, FilesEntitlement.Mode.READ))),
446448
new CreateClassLoaderEntitlement()
447449
)
448450
)

‎libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserFailureTests.java‎

Lines changed: 0 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -64,35 +64,6 @@ public void testEntitlementMissingDependentParameter() {
6464
);
6565
}
6666

67-
public void testEntitlementRelativePathWhenAbsolute() {
68-
PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream("""
69-
entitlement-module-name:
70-
- files:
71-
- path: test-path
72-
mode: read
73-
""".getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy());
74-
assertEquals(
75-
"[2:5] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] "
76-
+ "for entitlement type [files]: 'path' [test-path] must be absolute",
77-
ppe.getMessage()
78-
);
79-
}
80-
81-
public void testEntitlementAbsolutePathWhenRelative() {
82-
PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream("""
83-
entitlement-module-name:
84-
- files:
85-
- relative_path: /test-path
86-
relative_to: data
87-
mode: read
88-
""".getBytes(StandardCharsets.UTF_8)), "test-failure-policy.yaml", false).parsePolicy());
89-
assertEquals(
90-
"[2:5] policy parsing error for [test-failure-policy.yaml] in scope [entitlement-module-name] "
91-
+ "for entitlement type [files]: 'relative_path' [/test-path] must be relative",
92-
ppe.getMessage()
93-
);
94-
}
95-
9667
public void testEntitlementMutuallyExclusiveParameters() {
9768
PolicyParserException ppe = expectThrows(PolicyParserException.class, () -> new PolicyParser(new ByteArrayInputStream("""
9869
entitlement-module-name:

‎libs/entitlement/src/test/java/org/elasticsearch/entitlement/runtime/policy/PolicyParserTests.java‎

Lines changed: 55 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
package org.elasticsearch.entitlement.runtime.policy;
1111

12+
import org.elasticsearch.core.Strings;
1213
import org.elasticsearch.entitlement.runtime.policy.entitlements.CreateClassLoaderEntitlement;
1314
import org.elasticsearch.entitlement.runtime.policy.entitlements.Entitlement;
1415
import org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement;
@@ -18,18 +19,29 @@
1819
import org.elasticsearch.entitlement.runtime.policy.entitlements.SetHttpsConnectionPropertiesEntitlement;
1920
import org.elasticsearch.entitlement.runtime.policy.entitlements.WriteSystemPropertiesEntitlement;
2021
import org.elasticsearch.test.ESTestCase;
22+
import org.junit.BeforeClass;
2123

2224
import java.io.ByteArrayInputStream;
2325
import java.io.IOException;
26+
import java.io.InputStream;
2427
import java.nio.charset.StandardCharsets;
28+
import java.nio.file.Path;
2529
import java.util.List;
2630
import java.util.Map;
2731
import java.util.Set;
2832

2933
import static org.hamcrest.Matchers.equalTo;
3034

35+
@ESTestCase.WithoutSecurityManager
3136
public class PolicyParserTests extends ESTestCase {
3237

38+
public static String TEST_ABSOLUTE_PATH_TO_FILE;
39+
40+
@BeforeClass
41+
public static void beforeClass() throws IOException {
42+
TEST_ABSOLUTE_PATH_TO_FILE = createTempFile().toAbsolutePath().toString();
43+
}
44+
3345
private static class TestWrongEntitlementName implements Entitlement {}
3446

3547
public static class ManyConstructorsEntitlement implements Entitlement {
@@ -79,62 +91,65 @@ public void testGetEntitlementTypeName() {
7991
);
8092
}
8193

94+
private static InputStream createFilesTestPolicy() {
95+
return new ByteArrayInputStream(Strings.format("""
96+
entitlement-module-name:
97+
- files:
98+
- path: '%s'
99+
mode: "read_write"
100+
""", TEST_ABSOLUTE_PATH_TO_FILE).getBytes(StandardCharsets.UTF_8));
101+
}
102+
82103
public void testPolicyBuilder() throws IOException {
83-
Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml", false)
84-
.parsePolicy();
104+
Policy parsedPolicy = new PolicyParser(createFilesTestPolicy(), "test-policy.yaml", false).parsePolicy();
85105
Policy expected = new Policy(
86106
"test-policy.yaml",
87107
List.of(
88108
new Scope(
89109
"entitlement-module-name",
90-
List.of(FilesEntitlement.build(List.of(Map.of("path", "/test/path/to/file", "mode", "read_write"))))
110+
List.of(FilesEntitlement.build(List.of(Map.of("path", TEST_ABSOLUTE_PATH_TO_FILE, "mode", "read_write"))))
91111
)
92112
)
93113
);
94114
assertEquals(expected, parsedPolicy);
95115
}
96116

97117
public void testPolicyBuilderOnExternalPlugin() throws IOException {
98-
Policy parsedPolicy = new PolicyParser(PolicyParserTests.class.getResourceAsStream("test-policy.yaml"), "test-policy.yaml", true)
99-
.parsePolicy();
118+
Policy parsedPolicy = new PolicyParser(createFilesTestPolicy(), "test-policy.yaml", true).parsePolicy();
100119
Policy expected = new Policy(
101120
"test-policy.yaml",
102121
List.of(
103122
new Scope(
104123
"entitlement-module-name",
105-
List.of(FilesEntitlement.build(List.of(Map.of("path", "/test/path/to/file", "mode", "read_write"))))
124+
List.of(FilesEntitlement.build(List.of(Map.of("path", TEST_ABSOLUTE_PATH_TO_FILE, "mode", "read_write"))))
106125
)
107126
)
108127
);
109128
assertEquals(expected, parsedPolicy);
110129
}
111130

112131
public void testParseFiles() throws IOException {
113-
Policy policyWithOnePath = new PolicyParser(new ByteArrayInputStream("""
114-
entitlement-module-name:
115-
- files:
116-
- path: "/test/path/to/file"
117-
mode: "read_write"
118-
""".getBytes(StandardCharsets.UTF_8)), "test-policy.yaml", false).parsePolicy();
132+
Policy policyWithOnePath = new PolicyParser(createFilesTestPolicy(), "test-policy.yaml", false).parsePolicy();
119133
Policy expected = new Policy(
120134
"test-policy.yaml",
121135
List.of(
122136
new Scope(
123137
"entitlement-module-name",
124-
List.of(FilesEntitlement.build(List.of(Map.of("path", "/test/path/to/file", "mode", "read_write"))))
138+
List.of(FilesEntitlement.build(List.of(Map.of("path", TEST_ABSOLUTE_PATH_TO_FILE, "mode", "read_write"))))
125139
)
126140
)
127141
);
128142
assertEquals(expected, policyWithOnePath);
129143

130-
Policy policyWithTwoPaths = new PolicyParser(new ByteArrayInputStream("""
144+
String testPathToReadDir = createTempDir().toAbsolutePath().toString();
145+
Policy policyWithTwoPaths = new PolicyParser(new ByteArrayInputStream(Strings.format("""
131146
entitlement-module-name:
132147
- files:
133-
- path: "/test/path/to/file"
148+
- path: '%s'
134149
mode: "read_write"
135-
- path: "/test/path/to/read-dir/"
150+
- path: '%s'
136151
mode: "read"
137-
""".getBytes(StandardCharsets.UTF_8)), "test-policy.yaml", false).parsePolicy();
152+
""", TEST_ABSOLUTE_PATH_TO_FILE, testPathToReadDir).getBytes(StandardCharsets.UTF_8)), "test-policy.yaml", false).parsePolicy();
138153
expected = new Policy(
139154
"test-policy.yaml",
140155
List.of(
@@ -143,8 +158,8 @@ public void testParseFiles() throws IOException {
143158
List.of(
144159
FilesEntitlement.build(
145160
List.of(
146-
Map.of("path", "/test/path/to/file", "mode", "read_write"),
147-
Map.of("path", "/test/path/to/read-dir/", "mode", "read")
161+
Map.of("path", TEST_ABSOLUTE_PATH_TO_FILE, "mode", "read_write"),
162+
Map.of("path", testPathToReadDir, "mode", "read")
148163
)
149164
)
150165
)
@@ -153,18 +168,24 @@ public void testParseFiles() throws IOException {
153168
);
154169
assertEquals(expected, policyWithTwoPaths);
155170

156-
Policy policyWithMultiplePathsAndBaseDir = new PolicyParser(new ByteArrayInputStream("""
157-
entitlement-module-name:
158-
- files:
159-
- relative_path: "test/path/to/file"
160-
relative_to: "data"
161-
mode: "read_write"
162-
- relative_path: "test/path/to/read-dir/"
163-
relative_to: "config"
164-
mode: "read"
165-
- path: "/path/to/file"
166-
mode: "read_write"
167-
""".getBytes(StandardCharsets.UTF_8)), "test-policy.yaml", false).parsePolicy();
171+
String relativePathToFile = Path.of("test/path/to/file").normalize().toString();
172+
String relativePathToDir = Path.of("test/path/to/read-dir/").normalize().toString();
173+
Policy policyWithMultiplePathsAndBaseDir = new PolicyParser(
174+
new ByteArrayInputStream(Strings.format("""
175+
entitlement-module-name:
176+
- files:
177+
- relative_path: '%s'
178+
relative_to: "data"
179+
mode: "read_write"
180+
- relative_path: '%s'
181+
relative_to: "config"
182+
mode: "read"
183+
- path: '%s'
184+
mode: "read_write"
185+
""", relativePathToFile, relativePathToDir, TEST_ABSOLUTE_PATH_TO_FILE).getBytes(StandardCharsets.UTF_8)),
186+
"test-policy.yaml",
187+
false
188+
).parsePolicy();
168189
expected = new Policy(
169190
"test-policy.yaml",
170191
List.of(
@@ -173,9 +194,9 @@ public void testParseFiles() throws IOException {
173194
List.of(
174195
FilesEntitlement.build(
175196
List.of(
176-
Map.of("relative_path", "test/path/to/file", "mode", "read_write", "relative_to", "data"),
177-
Map.of("relative_path", "test/path/to/read-dir/", "mode", "read", "relative_to", "config"),
178-
Map.of("path", "/path/to/file", "mode", "read_write")
197+
Map.of("relative_path", relativePathToFile, "mode", "read_write", "relative_to", "data"),
198+
Map.of("relative_path", relativePathToDir, "mode", "read", "relative_to", "config"),
199+
Map.of("path", TEST_ABSOLUTE_PATH_TO_FILE, "mode", "read_write")
179200
)
180201
)
181202
)

‎libs/entitlement/src/test/resources/org/elasticsearch/entitlement/runtime/policy/test-policy.yaml‎

Lines changed: 0 additions & 4 deletions
This file was deleted.

‎muted-tests.yml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -134,9 +134,6 @@ tests:
134134
- class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT
135135
method: test {p0=data_stream/120_data_streams_stats/Multiple data stream}
136136
issue: https://github.com/elastic/elasticsearch/issues/118217
137-
- class: org.elasticsearch.action.search.SearchQueryThenFetchAsyncActionTests
138-
method: testBottomFieldSort
139-
issue: https://github.com/elastic/elasticsearch/issues/118214
140137
- class: org.elasticsearch.xpack.searchablesnapshots.RetrySearchIntegTests
141138
method: testSearcherId
142139
issue: https://github.com/elastic/elasticsearch/issues/118374

‎server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java‎

Lines changed: 0 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -256,32 +256,6 @@ void skipShard(SearchShardIterator iterator) {
256256
successfulShardExecution(iterator);
257257
}
258258

259-
private static boolean assertExecuteOnStartThread() {
260-
// Ensure that the current code has the following stacktrace:
261-
// AbstractSearchAsyncAction#start -> AbstractSearchAsyncAction#executePhase -> AbstractSearchAsyncAction#performPhaseOnShard
262-
final StackTraceElement[] stackTraceElements = Thread.currentThread().getStackTrace();
263-
assert stackTraceElements.length >= 6 : stackTraceElements;
264-
int index = 0;
265-
assert stackTraceElements[index++].getMethodName().equals("getStackTrace");
266-
assert stackTraceElements[index++].getMethodName().equals("assertExecuteOnStartThread");
267-
assert stackTraceElements[index++].getMethodName().equals("failOnUnavailable");
268-
if (stackTraceElements[index].getMethodName().equals("performPhaseOnShard")) {
269-
assert stackTraceElements[index].getClassName().endsWith("CanMatchPreFilterSearchPhase");
270-
index++;
271-
}
272-
assert stackTraceElements[index].getClassName().endsWith("AbstractSearchAsyncAction");
273-
assert stackTraceElements[index++].getMethodName().equals("run");
274-
275-
assert stackTraceElements[index].getClassName().endsWith("AbstractSearchAsyncAction");
276-
assert stackTraceElements[index++].getMethodName().equals("executePhase");
277-
278-
assert stackTraceElements[index].getClassName().endsWith("AbstractSearchAsyncAction");
279-
assert stackTraceElements[index++].getMethodName().equals("start");
280-
281-
assert stackTraceElements[index].getClassName().endsWith("AbstractSearchAsyncAction") == false;
282-
return true;
283-
}
284-
285259
private void performPhaseOnShard(final int shardIndex, final SearchShardIterator shardIt, final SearchShardTarget shard) {
286260
if (throttleConcurrentRequests) {
287261
var pendingExecutions = pendingExecutionsPerNode.computeIfAbsent(
@@ -323,7 +297,6 @@ public void onFailure(Exception e) {
323297
}
324298

325299
private void failOnUnavailable(int shardIndex, SearchShardIterator shardIt) {
326-
assert assertExecuteOnStartThread();
327300
SearchShardTarget unassignedShard = new SearchShardTarget(null, shardIt.shardId(), shardIt.getClusterAlias());
328301
onShardFailure(shardIndex, unassignedShard, shardIt, new NoShardAvailableActionException(shardIt.shardId()));
329302
}

‎server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import java.io.IOException;
2323
import java.util.ArrayList;
2424
import java.util.List;
25+
import java.util.Objects;
2526

2627
/**
2728
* Loads values from {@code _source}. This whole process is very slow and cast-tastic,
@@ -230,7 +231,7 @@ private static class BytesRefs extends BlockSourceReader {
230231

231232
@Override
232233
protected void append(BlockLoader.Builder builder, Object v) {
233-
((BlockLoader.BytesRefBuilder) builder).appendBytesRef(toBytesRef(scratch, (String) v));
234+
((BlockLoader.BytesRefBuilder) builder).appendBytesRef(toBytesRef(scratch, Objects.toString(v)));
234235
}
235236

236237
@Override

‎server/src/test/java/org/elasticsearch/action/search/SearchQueryThenFetchAsyncActionTests.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,6 @@ protected void run() {
220220
assertFalse(canReturnNullResponse.get());
221221
assertThat(numWithTopDocs.get(), equalTo(0));
222222
} else {
223-
assertTrue(canReturnNullResponse.get());
224223
if (withCollapse) {
225224
assertThat(numWithTopDocs.get(), equalTo(0));
226225
} else {

0 commit comments

Comments
 (0)