Skip to content

Commit ebb5050

Browse files
fang-xing-esqlfang-xing-esql
committed
Merge branch 'main' into double-question-marks
2 parents 13c7409 + ad99b0d commit ebb5050

File tree

300 files changed

+8619
-8384
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

300 files changed

+8619
-8384
lines changed

‎benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java‎

Lines changed: 55 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,14 @@
1111

1212
import org.apache.lucene.util.BytesRef;
1313
import org.elasticsearch.common.breaker.NoopCircuitBreaker;
14+
import org.elasticsearch.common.settings.Settings;
1415
import org.elasticsearch.common.util.BigArrays;
1516
import org.elasticsearch.compute.data.Block;
1617
import org.elasticsearch.compute.data.BlockFactory;
1718
import org.elasticsearch.compute.data.BooleanBlock;
1819
import org.elasticsearch.compute.data.BooleanVector;
20+
import org.elasticsearch.compute.data.BytesRefBlock;
21+
import org.elasticsearch.compute.data.BytesRefVector;
1922
import org.elasticsearch.compute.data.DoubleBlock;
2023
import org.elasticsearch.compute.data.DoubleVector;
2124
import org.elasticsearch.compute.data.LongBlock;
@@ -40,9 +43,13 @@
4043
import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin;
4144
import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce;
4245
import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
46+
import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToLower;
47+
import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToUpper;
4348
import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add;
4449
import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
4550
import org.elasticsearch.xpack.esql.planner.Layout;
51+
import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
52+
import org.elasticsearch.xpack.esql.session.Configuration;
4653
import org.openjdk.jmh.annotations.Benchmark;
4754
import org.openjdk.jmh.annotations.BenchmarkMode;
4855
import org.openjdk.jmh.annotations.Fork;
@@ -56,8 +63,10 @@
5663
import org.openjdk.jmh.annotations.Warmup;
5764

5865
import java.time.Duration;
66+
import java.time.ZoneOffset;
5967
import java.util.Arrays;
6068
import java.util.List;
69+
import java.util.Locale;
6170
import java.util.Map;
6271
import java.util.concurrent.TimeUnit;
6372

@@ -106,7 +115,9 @@ public class EvalBenchmark {
106115
"long_equal_to_int",
107116
"mv_min",
108117
"mv_min_ascending",
109-
"rlike" }
118+
"rlike",
119+
"to_lower",
120+
"to_upper" }
110121
)
111122
public String operation;
112123

@@ -169,7 +180,7 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
169180
new Coalesce(Source.EMPTY, lhs, List.of(f2)),
170181
layout(f1, f2)
171182
).get(driverContext);
172-
String desc = operation.endsWith("lazy") ? "CoalesceLazyEvaluator" : "CoalesceEagerEvaluator";
183+
String desc = operation.endsWith("lazy") ? "CoalesceLongLazyEvaluator" : "CoalesceLongEagerEvaluator";
173184
if (evaluator.toString().contains(desc) == false) {
174185
throw new IllegalArgumentException("Evaluator was [" + evaluator + "] but expected one containing [" + desc + "]");
175186
}
@@ -214,6 +225,16 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
214225
RLike rlike = new RLike(Source.EMPTY, keywordField, new RLikePattern(".ar"));
215226
yield EvalMapper.toEvaluator(FOLD_CONTEXT, rlike, layout(keywordField)).get(driverContext);
216227
}
228+
case "to_lower" -> {
229+
FieldAttribute keywordField = keywordField();
230+
ToLower toLower = new ToLower(Source.EMPTY, keywordField, configuration());
231+
yield EvalMapper.toEvaluator(FOLD_CONTEXT, toLower, layout(keywordField)).get(driverContext);
232+
}
233+
case "to_upper" -> {
234+
FieldAttribute keywordField = keywordField();
235+
ToUpper toUpper = new ToUpper(Source.EMPTY, keywordField, configuration());
236+
yield EvalMapper.toEvaluator(FOLD_CONTEXT, toUpper, layout(keywordField)).get(driverContext);
237+
}
217238
default -> throw new UnsupportedOperationException();
218239
};
219240
}
@@ -234,6 +255,23 @@ private static FieldAttribute keywordField() {
234255
return new FieldAttribute(Source.EMPTY, "keyword", new EsField("keyword", DataType.KEYWORD, Map.of(), true));
235256
}
236257

258+
private static Configuration configuration() {
259+
return new Configuration(
260+
ZoneOffset.UTC,
261+
Locale.ROOT,
262+
null,
263+
null,
264+
null,
265+
EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE.get(Settings.EMPTY),
266+
EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.get(Settings.EMPTY),
267+
null,
268+
false,
269+
Map.of(),
270+
0,
271+
false
272+
);
273+
}
274+
237275
private static Layout layout(FieldAttribute... fields) {
238276
Layout.Builder layout = new Layout.Builder();
239277
layout.append(Arrays.asList(fields));
@@ -366,10 +404,24 @@ private static void checkExpected(String operation, Page actual) {
366404
}
367405
}
368406
}
407+
case "to_lower" -> checkBytes(operation, actual, new BytesRef[] { new BytesRef("foo"), new BytesRef("bar") });
408+
case "to_upper" -> checkBytes(operation, actual, new BytesRef[] { new BytesRef("FOO"), new BytesRef("BAR") });
369409
default -> throw new UnsupportedOperationException(operation);
370410
}
371411
}
372412

413+
private static void checkBytes(String operation, Page actual, BytesRef[] expectedVals) {
414+
BytesRef scratch = new BytesRef();
415+
BytesRefVector v = actual.<BytesRefBlock>getBlock(1).asVector();
416+
for (int i = 0; i < BLOCK_LENGTH; i++) {
417+
BytesRef expected = expectedVals[i % 2];
418+
BytesRef b = v.getBytesRef(i, scratch);
419+
if (b.equals(expected) == false) {
420+
throw new AssertionError("[" + operation + "] expected [" + expected + "] but was [" + b + "]");
421+
}
422+
}
423+
}
424+
373425
private static Page page(String operation) {
374426
return switch (operation) {
375427
case "abs", "add", "date_trunc", "equal_to_const" -> {
@@ -440,7 +492,7 @@ private static Page page(String operation) {
440492
}
441493
yield new Page(builder.build());
442494
}
443-
case "rlike" -> {
495+
case "rlike", "to_lower", "to_upper" -> {
444496
var builder = blockFactory.newBytesRefVectorBuilder(BLOCK_LENGTH);
445497
BytesRef[] values = new BytesRef[] { new BytesRef("foo"), new BytesRef("bar") };
446498
for (int i = 0; i < BLOCK_LENGTH; i++) {

‎docs/changelog/121920.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 121920
2+
summary: Account for the `SearchHit` source in circuit breaker
3+
area: Search
4+
type: enhancement
5+
issues:
6+
- 89656

‎docs/changelog/122860.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 122860
2+
summary: Improved error message when index field type is invalid
3+
area: Mapping
4+
type: enhancement
5+
issues: []

‎docs/changelog/123105.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 123105
2+
summary: fix stale data in synthetic source for string stored field
3+
area: Mapping
4+
type: bug
5+
issues:
6+
- 123110

‎docs/changelog/123197.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 123197
2+
summary: Fix early termination in `LuceneSourceOperator`
3+
area: ES|QL
4+
type: bug
5+
issues: []

‎docs/changelog/123272.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 123272
2+
summary: Set Connect Timeout to 5s
3+
area: Machine Learning
4+
type: bug
5+
issues: []

‎libs/entitlement/src/main/java/org/elasticsearch/entitlement/bootstrap/EntitlementBootstrap.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,8 @@ public record BootstrapArgs(
3939
Function<Class<?>, String> pluginResolver,
4040
Function<String, String> settingResolver,
4141
Function<String, Stream<String>> settingGlobResolver,
42-
Function<String, Path> repoDirResolver,
4342
Path[] dataDirs,
43+
Path[] sharedRepoDirs,
4444
Path configDir,
4545
Path libDir,
4646
Path logsDir,
@@ -51,11 +51,11 @@ public record BootstrapArgs(
5151
requireNonNull(pluginResolver);
5252
requireNonNull(settingResolver);
5353
requireNonNull(settingGlobResolver);
54-
requireNonNull(repoDirResolver);
5554
requireNonNull(dataDirs);
5655
if (dataDirs.length == 0) {
5756
throw new IllegalArgumentException("must provide at least one data directory");
5857
}
58+
requireNonNull(sharedRepoDirs);
5959
requireNonNull(configDir);
6060
requireNonNull(libDir);
6161
requireNonNull(logsDir);
@@ -77,8 +77,8 @@ public static BootstrapArgs bootstrapArgs() {
7777
* @param pluginResolver a functor to map a Java Class to the plugin it belongs to (the plugin name).
7878
* @param settingResolver a functor to resolve the value of an Elasticsearch setting.
7979
* @param settingGlobResolver a functor to resolve a glob expression for one or more Elasticsearch settings.
80-
* @param repoDirResolver a functor to map a repository location to its Elasticsearch path.
8180
* @param dataDirs data directories for Elasticsearch
81+
* @param sharedRepoDirs shared repository directories for Elasticsearch
8282
* @param configDir the config directory for Elasticsearch
8383
* @param libDir the lib directory for Elasticsearch
8484
* @param tempDir the temp directory for Elasticsearch
@@ -89,8 +89,8 @@ public static void bootstrap(
8989
Function<Class<?>, String> pluginResolver,
9090
Function<String, String> settingResolver,
9191
Function<String, Stream<String>> settingGlobResolver,
92-
Function<String, Path> repoDirResolver,
9392
Path[] dataDirs,
93+
Path[] sharedRepoDirs,
9494
Path configDir,
9595
Path libDir,
9696
Path logsDir,
@@ -105,8 +105,8 @@ public static void bootstrap(
105105
pluginResolver,
106106
settingResolver,
107107
settingGlobResolver,
108-
repoDirResolver,
109108
dataDirs,
109+
sharedRepoDirs,
110110
configDir,
111111
libDir,
112112
logsDir,

‎libs/entitlement/src/main/java/org/elasticsearch/entitlement/initialization/EntitlementInitialization.java‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,8 @@
6363
import java.util.stream.Stream;
6464
import java.util.stream.StreamSupport;
6565

66+
import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.BaseDir.DATA;
67+
import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.BaseDir.SHARED_REPO;
6668
import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Mode.READ;
6769
import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.Mode.READ_WRITE;
6870

@@ -138,6 +140,7 @@ private static PolicyManager createPolicyManager() {
138140
getUserHome(),
139141
bootstrapArgs.configDir(),
140142
bootstrapArgs.dataDirs(),
143+
bootstrapArgs.sharedRepoDirs(),
141144
bootstrapArgs.tempDir(),
142145
bootstrapArgs.settingResolver(),
143146
bootstrapArgs.settingGlobResolver()
@@ -152,8 +155,8 @@ private static PolicyManager createPolicyManager() {
152155
new CreateClassLoaderEntitlement(),
153156
new FilesEntitlement(
154157
List.of(
155-
FileData.ofPath(bootstrapArgs.repoDirResolver().apply(""), READ_WRITE),
156-
FileData.ofRelativePath(Path.of(""), FilesEntitlement.BaseDir.DATA, READ_WRITE)
158+
FileData.ofRelativePath(Path.of(""), SHARED_REPO, READ_WRITE),
159+
FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE)
157160
)
158161
)
159162
)
@@ -175,8 +178,8 @@ private static PolicyManager createPolicyManager() {
175178
FileData.ofPath(bootstrapArgs.tempDir(), READ_WRITE),
176179
FileData.ofPath(bootstrapArgs.configDir(), READ),
177180
FileData.ofPath(bootstrapArgs.logsDir(), READ_WRITE),
178-
FileData.ofRelativePath(Path.of(""), FilesEntitlement.BaseDir.DATA, READ_WRITE),
179-
FileData.ofPath(bootstrapArgs.repoDirResolver().apply(""), READ_WRITE),
181+
FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE),
182+
FileData.ofRelativePath(Path.of(""), SHARED_REPO, READ_WRITE),
180183

181184
// OS release on Linux
182185
FileData.ofPath(Path.of("/etc/os-release"), READ),
@@ -210,21 +213,21 @@ private static PolicyManager createPolicyManager() {
210213
List.of(
211214
FileData.ofPath(bootstrapArgs.configDir(), READ),
212215
FileData.ofPath(bootstrapArgs.tempDir(), READ),
213-
FileData.ofRelativePath(Path.of(""), FilesEntitlement.BaseDir.DATA, READ_WRITE)
216+
FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE)
214217
)
215218
)
216219
)
217220
),
218221
new Scope(
219222
"org.apache.lucene.misc",
220-
List.of(new FilesEntitlement(List.of(FileData.ofRelativePath(Path.of(""), FilesEntitlement.BaseDir.DATA, READ_WRITE))))
223+
List.of(new FilesEntitlement(List.of(FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE))))
221224
),
222225
new Scope("org.apache.logging.log4j.core", List.of(new ManageThreadsEntitlement())),
223226
new Scope(
224227
"org.elasticsearch.nativeaccess",
225228
List.of(
226229
new LoadNativeLibrariesEntitlement(),
227-
new FilesEntitlement(List.of(FileData.ofRelativePath(Path.of(""), FilesEntitlement.BaseDir.DATA, READ_WRITE)))
230+
new FilesEntitlement(List.of(FileData.ofRelativePath(Path.of(""), DATA, READ_WRITE)))
228231
)
229232
)
230233
);

‎libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/FileAccessTree.java‎

Lines changed: 41 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import java.nio.file.Path;
1515
import java.util.ArrayList;
1616
import java.util.Arrays;
17+
import java.util.Comparator;
1718
import java.util.List;
1819
import java.util.Objects;
1920

@@ -46,8 +47,8 @@ private FileAccessTree(FilesEntitlement filesEntitlement, PathLookup pathLookup)
4647
readPaths.add(tempDir);
4748
writePaths.add(tempDir);
4849

49-
readPaths.sort(String::compareTo);
50-
writePaths.sort(String::compareTo);
50+
readPaths.sort(PATH_ORDER);
51+
writePaths.sort(PATH_ORDER);
5152

5253
this.readPaths = pruneSortedPaths(readPaths).toArray(new String[0]);
5354
this.writePaths = pruneSortedPaths(writePaths).toArray(new String[0]);
@@ -60,7 +61,7 @@ private static List<String> pruneSortedPaths(List<String> paths) {
6061
prunedReadPaths.add(currentPath);
6162
for (int i = 1; i < paths.size(); ++i) {
6263
String nextPath = paths.get(i);
63-
if (nextPath.startsWith(currentPath) == false) {
64+
if (isParent(currentPath, nextPath) == false) {
6465
prunedReadPaths.add(nextPath);
6566
currentPath = nextPath;
6667
}
@@ -88,21 +89,28 @@ static String normalizePath(Path path) {
8889
// Note that toAbsolutePath produces paths separated by the default file separator,
8990
// so on Windows, if the given path uses forward slashes, this consistently
9091
// converts it to backslashes.
91-
return path.toAbsolutePath().normalize().toString();
92+
String result = path.toAbsolutePath().normalize().toString();
93+
while (result.endsWith(FILE_SEPARATOR)) {
94+
result = result.substring(0, result.length() - FILE_SEPARATOR.length());
95+
}
96+
return result;
9297
}
9398

9499
private static boolean checkPath(String path, String[] paths) {
95100
if (paths.length == 0) {
96101
return false;
97102
}
98-
int ndx = Arrays.binarySearch(paths, path);
103+
int ndx = Arrays.binarySearch(paths, path, PATH_ORDER);
99104
if (ndx < -1) {
100-
String maybeParent = paths[-ndx - 2];
101-
return path.startsWith(maybeParent) && path.startsWith(FILE_SEPARATOR, maybeParent.length());
105+
return isParent(paths[-ndx - 2], path);
102106
}
103107
return ndx >= 0;
104108
}
105109

110+
private static boolean isParent(String maybeParent, String path) {
111+
return path.startsWith(maybeParent) && path.startsWith(FILE_SEPARATOR, maybeParent.length());
112+
}
113+
106114
@Override
107115
public boolean equals(Object o) {
108116
if (o == null || getClass() != o.getClass()) return false;
@@ -114,4 +122,30 @@ public boolean equals(Object o) {
114122
public int hashCode() {
115123
return Objects.hash(Arrays.hashCode(readPaths), Arrays.hashCode(writePaths));
116124
}
125+
126+
/**
127+
* For our lexicographic sort trick to work correctly, we must have path separators sort before
128+
* any other character so that files in a directory appear immediately after that directory.
129+
* For example, we require [/a, /a/b, /a.xml] rather than the natural order [/a, /a.xml, /a/b].
130+
*/
131+
private static final Comparator<String> PATH_ORDER = (s1, s2) -> {
132+
Path p1 = Path.of(s1);
133+
Path p2 = Path.of(s2);
134+
var i1 = p1.iterator();
135+
var i2 = p2.iterator();
136+
while (i1.hasNext() && i2.hasNext()) {
137+
int cmp = i1.next().compareTo(i2.next());
138+
if (cmp != 0) {
139+
return cmp;
140+
}
141+
}
142+
if (i1.hasNext()) {
143+
return 1;
144+
} else if (i2.hasNext()) {
145+
return -1;
146+
} else {
147+
assert p1.equals(p2);
148+
return 0;
149+
}
150+
};
117151
}

‎libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PathLookup.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ public record PathLookup(
1717
Path homeDir,
1818
Path configDir,
1919
Path[] dataDirs,
20+
Path[] sharedRepoDirs,
2021
Path tempDir,
2122
Function<String, String> settingResolver,
2223
Function<String, Stream<String>> settingGlobResolver

0 commit comments

Comments
 (0)