Skip to content

Commit 8f7778e

Browse files
nik9000nik9000
authored
Merge branch 'main' into esql_disable_import_on_gen
2 parents ba49b86 + 9801d45 commit 8f7778e

File tree

10 files changed

+155
-11
lines changed

10 files changed

+155
-11
lines changed

‎muted-tests.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -444,6 +444,9 @@ tests:
444444
- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
445445
method: test {rerank.Reranker using another sort order ASYNC}
446446
issue: https://github.com/elastic/elasticsearch/issues/127638
447+
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapperTests
448+
method: testRescoreVectorOldIndexVersion
449+
issue: https://github.com/elastic/elasticsearch/issues/127657
447450

448451
# Examples:
449452
#

‎server/src/main/java/org/elasticsearch/index/IndexVersions.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ private static Version parseUnchecked(String version) {
137137
public static final IndexVersion LOGSB_OPTIONAL_SORTING_ON_HOST_NAME_BACKPORT = def(8_525_0_00, parseUnchecked("9.12.1"));
138138
public static final IndexVersion USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT_BACKPORT = def(8_526_0_00, parseUnchecked("9.12.1"));
139139
public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_BACKPORT_8_X = def(8_527_0_00, Version.LUCENE_9_12_1);
140+
public static final IndexVersion ADD_RESCORE_PARAMS_TO_QUANTIZED_VECTORS_BACKPORT_8_X = def(8_528_0_00, Version.LUCENE_9_12_1);
140141
public static final IndexVersion UPGRADE_TO_LUCENE_10_0_0 = def(9_000_0_00, Version.LUCENE_10_0_0);
141142
public static final IndexVersion LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT = def(9_001_0_00, Version.LUCENE_10_0_0);
142143
public static final IndexVersion TIME_BASED_K_ORDERED_DOC_ID = def(9_002_0_00, Version.LUCENE_10_0_0);

‎server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,11 @@ public static boolean isNotUnitVector(float magnitude) {
109109
return Math.abs(magnitude - 1.0f) > EPS;
110110
}
111111

112+
private static boolean hasRescoreIndexVersion(IndexVersion version) {
113+
return version.onOrAfter(IndexVersions.ADD_RESCORE_PARAMS_TO_QUANTIZED_VECTORS)
114+
|| version.between(IndexVersions.ADD_RESCORE_PARAMS_TO_QUANTIZED_VECTORS_BACKPORT_8_X, IndexVersions.UPGRADE_TO_LUCENE_10_0_0);
115+
}
116+
112117
public static final IndexVersion MAGNITUDE_STORED_INDEX_VERSION = IndexVersions.V_7_5_0;
113118
public static final IndexVersion INDEXED_BY_DEFAULT_INDEX_VERSION = IndexVersions.FIRST_DETACHED_INDEX_VERSION;
114119
public static final IndexVersion NORMALIZE_COSINE = IndexVersions.NORMALIZED_VECTOR_COSINE;
@@ -1350,7 +1355,7 @@ public IndexOptions parseIndexOptions(String fieldName, Map<String, ?> indexOpti
13501355
confidenceInterval = (float) XContentMapValues.nodeDoubleValue(confidenceIntervalNode);
13511356
}
13521357
RescoreVector rescoreVector = null;
1353-
if (indexVersion.onOrAfter(ADD_RESCORE_PARAMS_TO_QUANTIZED_VECTORS)) {
1358+
if (hasRescoreIndexVersion(indexVersion)) {
13541359
rescoreVector = RescoreVector.fromIndexOptions(indexOptionsMap, indexVersion);
13551360
}
13561361
MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap);
@@ -1385,7 +1390,7 @@ public IndexOptions parseIndexOptions(String fieldName, Map<String, ?> indexOpti
13851390
confidenceInterval = (float) XContentMapValues.nodeDoubleValue(confidenceIntervalNode);
13861391
}
13871392
RescoreVector rescoreVector = null;
1388-
if (indexVersion.onOrAfter(ADD_RESCORE_PARAMS_TO_QUANTIZED_VECTORS)) {
1393+
if (hasRescoreIndexVersion(indexVersion)) {
13891394
rescoreVector = RescoreVector.fromIndexOptions(indexOptionsMap, indexVersion);
13901395
}
13911396
MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap);
@@ -1428,7 +1433,7 @@ public IndexOptions parseIndexOptions(String fieldName, Map<String, ?> indexOpti
14281433
confidenceInterval = (float) XContentMapValues.nodeDoubleValue(confidenceIntervalNode);
14291434
}
14301435
RescoreVector rescoreVector = null;
1431-
if (indexVersion.onOrAfter(ADD_RESCORE_PARAMS_TO_QUANTIZED_VECTORS)) {
1436+
if (hasRescoreIndexVersion(indexVersion)) {
14321437
rescoreVector = RescoreVector.fromIndexOptions(indexOptionsMap, indexVersion);
14331438
}
14341439
MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap);
@@ -1454,7 +1459,7 @@ public IndexOptions parseIndexOptions(String fieldName, Map<String, ?> indexOpti
14541459
confidenceInterval = (float) XContentMapValues.nodeDoubleValue(confidenceIntervalNode);
14551460
}
14561461
RescoreVector rescoreVector = null;
1457-
if (indexVersion.onOrAfter(ADD_RESCORE_PARAMS_TO_QUANTIZED_VECTORS)) {
1462+
if (hasRescoreIndexVersion(indexVersion)) {
14581463
rescoreVector = RescoreVector.fromIndexOptions(indexOptionsMap, indexVersion);
14591464
}
14601465
MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap);
@@ -1485,7 +1490,7 @@ public IndexOptions parseIndexOptions(String fieldName, Map<String, ?> indexOpti
14851490
int m = XContentMapValues.nodeIntegerValue(mNode);
14861491
int efConstruction = XContentMapValues.nodeIntegerValue(efConstructionNode);
14871492
RescoreVector rescoreVector = null;
1488-
if (indexVersion.onOrAfter(ADD_RESCORE_PARAMS_TO_QUANTIZED_VECTORS)) {
1493+
if (hasRescoreIndexVersion(indexVersion)) {
14891494
rescoreVector = RescoreVector.fromIndexOptions(indexOptionsMap, indexVersion);
14901495
if (rescoreVector == null && indexVersion.onOrAfter(DEFAULT_OVERSAMPLE_VALUE_FOR_BBQ)) {
14911496
rescoreVector = new RescoreVector(DEFAULT_OVERSAMPLE);
@@ -1509,7 +1514,7 @@ public boolean supportsDimension(int dims) {
15091514
@Override
15101515
public IndexOptions parseIndexOptions(String fieldName, Map<String, ?> indexOptionsMap, IndexVersion indexVersion) {
15111516
RescoreVector rescoreVector = null;
1512-
if (indexVersion.onOrAfter(ADD_RESCORE_PARAMS_TO_QUANTIZED_VECTORS)) {
1517+
if (hasRescoreIndexVersion(indexVersion)) {
15131518
rescoreVector = RescoreVector.fromIndexOptions(indexOptionsMap, indexVersion);
15141519
if (rescoreVector == null && indexVersion.onOrAfter(DEFAULT_OVERSAMPLE_VALUE_FOR_BBQ)) {
15151520
rescoreVector = new RescoreVector(DEFAULT_OVERSAMPLE);

‎server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -906,8 +906,12 @@ public void testRescoreVectorForNonQuantized() {
906906
public void testRescoreVectorOldIndexVersion() {
907907
IndexVersion incompatibleVersion = IndexVersionUtils.randomVersionBetween(
908908
random(),
909-
IndexVersionUtils.getLowestReadCompatibleVersion(),
910-
IndexVersionUtils.getPreviousVersion(DenseVectorFieldMapper.ADD_RESCORE_PARAMS_TO_QUANTIZED_VECTORS)
909+
IndexVersionUtils.randomVersionBetween(
910+
random(),
911+
IndexVersionUtils.getLowestReadCompatibleVersion(),
912+
IndexVersionUtils.getPreviousVersion(IndexVersions.ADD_RESCORE_PARAMS_TO_QUANTIZED_VECTORS_BACKPORT_8_X)
913+
),
914+
IndexVersionUtils.getPreviousVersion(IndexVersions.ADD_RESCORE_PARAMS_TO_QUANTIZED_VECTORS)
911915
);
912916
for (String indexType : List.of("int8_hnsw", "int8_flat", "int4_hnsw", "int4_flat", "bbq_hnsw", "bbq_flat")) {
913917
expectThrows(

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries.csv-spec‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,24 @@ cost:double | cluster:keyword | time_bucket:datetime
184184
22.75 | qa | 2024-05-10T00:08:00.000Z
185185
;
186186

187+
min_over_time
188+
required_capability: metrics_command
189+
required_capability: min_over_time
190+
TS k8s | STATS cost=sum(min_over_time(network.cost)) BY cluster, time_bucket = bucket(@timestamp,1minute) | SORT cost DESC, time_bucket DESC, cluster | LIMIT 10;
191+
192+
cost:double | cluster:keyword | time_bucket:datetime
193+
29.0 | prod | 2024-05-10T00:19:00.000Z
194+
27.625 | qa | 2024-05-10T00:06:00.000Z
195+
24.25 | qa | 2024-05-10T00:09:00.000Z
196+
23.125 | staging | 2024-05-10T00:08:00.000Z
197+
22.5 | prod | 2024-05-10T00:13:00.000Z
198+
18.625 | qa | 2024-05-10T00:04:00.000Z
199+
18.0 | qa | 2024-05-10T00:11:00.000Z
200+
17.75 | qa | 2024-05-10T00:01:00.000Z
201+
17.125 | qa | 2024-05-10T00:22:00.000Z
202+
16.875 | qa | 2024-05-10T00:20:00.000Z
203+
;
204+
187205
max_of_avg_over_time
188206
required_capability: metrics_command
189207
required_capability: avg_over_time

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1054,7 +1054,12 @@ public enum Cap {
10541054
/**
10551055
* Guards a bug fix matching {@code TO_LOWER(f) == ""}.
10561056
*/
1057-
TO_LOWER_EMPTY_STRING;
1057+
TO_LOWER_EMPTY_STRING,
1058+
1059+
/**
1060+
* Support min_over_time aggregation that gets evaluated per time-series
1061+
*/
1062+
MIN_OVER_TIME(Build.current().isSnapshot());
10581063

10591064
private final boolean enabled;
10601065

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.elasticsearch.xpack.esql.expression.function.aggregate.Median;
2929
import org.elasticsearch.xpack.esql.expression.function.aggregate.MedianAbsoluteDeviation;
3030
import org.elasticsearch.xpack.esql.expression.function.aggregate.Min;
31+
import org.elasticsearch.xpack.esql.expression.function.aggregate.MinOverTime;
3132
import org.elasticsearch.xpack.esql.expression.function.aggregate.Percentile;
3233
import org.elasticsearch.xpack.esql.expression.function.aggregate.Rate;
3334
import org.elasticsearch.xpack.esql.expression.function.aggregate.SpatialCentroid;
@@ -444,6 +445,7 @@ private static FunctionDefinition[][] snapshotFunctions() {
444445
def(Delay.class, Delay::new, "delay"),
445446
def(Rate.class, Rate::withUnresolvedTimestamp, "rate"),
446447
def(MaxOverTime.class, uni(MaxOverTime::new), "max_over_time"),
448+
def(MinOverTime.class, uni(MinOverTime::new), "min_over_time"),
447449
def(AvgOverTime.class, uni(AvgOverTime::new), "avg_over_time"),
448450
def(LastOverTime.class, LastOverTime::withUnresolvedTimestamp, "last_over_time"),
449451
def(Term.class, bi(Term::new), "term") } };

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AggregateWritables.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
3030
Sum.ENTRY,
3131
Top.ENTRY,
3232
Values.ENTRY,
33+
MinOverTime.ENTRY,
3334
MaxOverTime.ENTRY,
3435
AvgOverTime.ENTRY,
3536
LastOverTime.ENTRY,

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinOverTime.java‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.esql.expression.function.aggregate;
9+
10+
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
11+
import org.elasticsearch.common.io.stream.StreamInput;
12+
import org.elasticsearch.xpack.esql.core.expression.Expression;
13+
import org.elasticsearch.xpack.esql.core.expression.Literal;
14+
import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
15+
import org.elasticsearch.xpack.esql.core.tree.Source;
16+
import org.elasticsearch.xpack.esql.core.type.DataType;
17+
import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
18+
import org.elasticsearch.xpack.esql.expression.function.FunctionType;
19+
import org.elasticsearch.xpack.esql.expression.function.Param;
20+
21+
import java.io.IOException;
22+
import java.util.List;
23+
24+
import static java.util.Collections.emptyList;
25+
26+
/**
27+
* Similar to {@link Min}, but it is used to calculate the minimum value over a time series of values from the given field.
28+
*/
29+
public class MinOverTime extends TimeSeriesAggregateFunction {
30+
public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
31+
Expression.class,
32+
"MinOverTime",
33+
MinOverTime::new
34+
);
35+
36+
@FunctionInfo(
37+
returnType = { "boolean", "double", "integer", "long", "date", "date_nanos", "ip", "keyword", "long", "version" },
38+
description = "The minimum over time value of a field.",
39+
type = FunctionType.AGGREGATE
40+
)
41+
public MinOverTime(
42+
Source source,
43+
@Param(
44+
name = "field",
45+
type = {
46+
"aggregate_metric_double",
47+
"boolean",
48+
"double",
49+
"integer",
50+
"long",
51+
"date",
52+
"date_nanos",
53+
"ip",
54+
"keyword",
55+
"text",
56+
"long",
57+
"version" }
58+
) Expression field
59+
) {
60+
this(source, field, Literal.TRUE);
61+
}
62+
63+
public MinOverTime(Source source, Expression field, Expression filter) {
64+
super(source, field, filter, emptyList());
65+
}
66+
67+
private MinOverTime(StreamInput in) throws IOException {
68+
super(in);
69+
}
70+
71+
@Override
72+
public String getWriteableName() {
73+
return ENTRY.name;
74+
}
75+
76+
@Override
77+
public MinOverTime withFilter(Expression filter) {
78+
return new MinOverTime(source(), field(), filter);
79+
}
80+
81+
@Override
82+
protected NodeInfo<MinOverTime> info() {
83+
return NodeInfo.create(this, MinOverTime::new, field(), filter());
84+
}
85+
86+
@Override
87+
public MinOverTime replaceChildren(List<Expression> newChildren) {
88+
return new MinOverTime(source(), newChildren.get(0), newChildren.get(1));
89+
}
90+
91+
@Override
92+
protected TypeResolution resolveType() {
93+
return perTimeSeriesAggregation().resolveType();
94+
}
95+
96+
@Override
97+
public DataType dataType() {
98+
return perTimeSeriesAggregation().dataType();
99+
}
100+
101+
@Override
102+
public Min perTimeSeriesAggregation() {
103+
return new Min(source(), field(), filter());
104+
}
105+
}

‎x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/60_usage.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ setup:
3333
path: /_query
3434
parameters: []
3535
# A snapshot function was removed in match_function_options, it can't work on mixed cluster tests otherwise.
36-
capabilities: [ snapshot_test_for_telemetry, fn_byte_length, match_function_options, last_over_time]
36+
capabilities: [ snapshot_test_for_telemetry, fn_byte_length, match_function_options, min_over_time ]
3737
reason: "Test that should only be executed on snapshot versions"
3838

3939
- do: {xpack.usage: {}}
@@ -123,7 +123,7 @@ setup:
123123
- match: {esql.functions.coalesce: $functions_coalesce}
124124
- gt: {esql.functions.categorize: $functions_categorize}
125125
# Testing for the entire function set isn't feasbile, so we just check that we return the correct count as an approximation.
126-
- length: {esql.functions: 138} # check the "sister" test below for a likely update to the same esql.functions length check
126+
- length: {esql.functions: 139} # check the "sister" test below for a likely update to the same esql.functions length check
127127

128128
---
129129
"Basic ESQL usage output (telemetry) non-snapshot version":

0 commit comments

Comments
 (0)