Skip to content

Commit 81dfc08

Browse files
felixbarnyfelixbarny
authored
PromQL: auto-bucketing for step (elastic#142582)
* default to 100 buckets when start/end is set but step is absent * customizable number of buckets with `buckets` promql command parameter
1 parent 4ab9e13 commit 81dfc08

File tree

9 files changed

+237
-59
lines changed

9 files changed

+237
-59
lines changed

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-timeseries-promql.csv-spec‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,21 @@ sum(avg_over_time(network.cost[5m])):double | step:date
2020
50.25 | 2024-05-10T00:20:00.000Z
2121
;
2222

23+
auto_step_buckets
24+
required_capability: promql_command_v0
25+
required_capability: promql_buckets_parameter
26+
PROMQL index=k8s start="2024-05-10T00:00:00.000Z" end="2024-05-10T00:20:00.000Z" buckets=4 1
27+
| KEEP step
28+
| SORT step;
29+
30+
step:datetime
31+
2024-05-10T00:00:00.000Z
32+
2024-05-10T00:05:00.000Z
33+
2024-05-10T00:10:00.000Z
34+
2024-05-10T00:15:00.000Z
35+
2024-05-10T00:20:00.000Z
36+
;
37+
2338
not_equals_filter
2439
required_capability: promql_command_v0
2540
PROMQL index=k8s step=1h cost=(max by (cluster) (network.total_bytes_in{cluster!="prod"}))

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1916,6 +1916,11 @@ public enum Cap {
19161916
*/
19171917
PROMQL_TIME,
19181918

1919+
/**
1920+
* Support for deriving PromQL time buckets from [start, end, buckets] when [step] is omitted.
1921+
*/
1922+
PROMQL_BUCKETS_PARAMETER,
1923+
19191924
/**
19201925
* Queries for unmapped fields return no data instead of an error.
19211926
* Also filters out nulls from results.

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/promql/TranslatePromqlToEsqlPlan.java‎

Lines changed: 30 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.elasticsearch.xpack.esql.core.expression.Attribute;
1414
import org.elasticsearch.xpack.esql.core.expression.Expression;
1515
import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
16+
import org.elasticsearch.xpack.esql.core.expression.FoldContext;
1617
import org.elasticsearch.xpack.esql.core.expression.Literal;
1718
import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
1819
import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
@@ -565,13 +566,7 @@ private static LogicalPlan convertValueToDouble(PromqlCommand promqlCommand, Log
565566
}
566567

567568
private static Alias createStepBucketAlias(PromqlCommand promqlCommand) {
568-
Expression timeBucketSize;
569-
if (promqlCommand.isRangeQuery()) {
570-
timeBucketSize = promqlCommand.step();
571-
} else {
572-
// use default lookback for instant queries
573-
timeBucketSize = Literal.timeDuration(promqlCommand.source(), DEFAULT_LOOKBACK);
574-
}
569+
Expression timeBucketSize = resolveTimeBucketSize(promqlCommand);
575570
Bucket b = new Bucket(
576571
timeBucketSize.source(),
577572
promqlCommand.timestamp(),
@@ -583,6 +578,34 @@ private static Alias createStepBucketAlias(PromqlCommand promqlCommand) {
583578
return new Alias(b.source(), STEP_COLUMN_NAME, b, promqlCommand.stepId());
584579
}
585580

581+
private static Expression resolveTimeBucketSize(PromqlCommand promqlCommand) {
582+
if (promqlCommand.isRangeQuery()) {
583+
if (promqlCommand.step().value() != null) {
584+
return promqlCommand.step();
585+
}
586+
return resolveAutoStepFromBuckets(promqlCommand);
587+
}
588+
// use default lookback for instant queries
589+
return Literal.timeDuration(promqlCommand.source(), DEFAULT_LOOKBACK);
590+
}
591+
592+
private static Literal resolveAutoStepFromBuckets(PromqlCommand promqlCommand) {
593+
Bucket autoBucket = new Bucket(
594+
promqlCommand.buckets().source(),
595+
promqlCommand.timestamp(),
596+
promqlCommand.buckets(),
597+
promqlCommand.start(),
598+
promqlCommand.end(),
599+
ConfigurationAware.CONFIGURATION_MARKER
600+
);
601+
long rangeStart = ((Number) promqlCommand.start().value()).longValue();
602+
long rangeEnd = ((Number) promqlCommand.end().value()).longValue();
603+
var rounding = autoBucket.getDateRounding(FoldContext.small(), rangeStart, rangeEnd);
604+
long roundedStart = rounding.round(rangeStart);
605+
long nextRoundedValue = rounding.nextRoundingValue(roundedStart);
606+
return Literal.timeDuration(promqlCommand.source(), Duration.ofMillis(Math.max(1L, nextRoundedValue - roundedStart)));
607+
}
608+
586609
/**
587610
* Translates PromQL label matchers into ESQL filter expressions.
588611
* <p>

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/LogicalPlanBuilder.java‎

Lines changed: 57 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -122,8 +122,9 @@
122122
*/
123123
public class LogicalPlanBuilder extends ExpressionBuilder {
124124

125-
private static final String TIME = "time", START = "start", END = "end", STEP = "step", INDEX = "index";
126-
private static final Set<String> PROMQL_ALLOWED_PARAMS = Set.of(TIME, START, END, STEP, INDEX);
125+
private static final String TIME = "time", START = "start", END = "end", STEP = "step", BUCKETS = "buckets", INDEX = "index";
126+
private static final int DEFAULT_PROMQL_BUCKETS = 100;
127+
private static final Set<String> PROMQL_ALLOWED_PARAMS = Set.of(TIME, START, END, STEP, BUCKETS, INDEX);
127128

128129
/**
129130
* Maximum number of commands allowed per query
@@ -1296,6 +1297,7 @@ public LogicalPlan visitPromqlCommand(EsqlBaseParser.PromqlCommandContext ctx) {
12961297
params.startLiteral(),
12971298
params.endLiteral(),
12981299
params.stepLiteral(),
1300+
params.bucketsLiteral(),
12991301
valueColumnName,
13001302
new UnresolvedTimestamp(source)
13011303
);
@@ -1318,6 +1320,7 @@ private PromqlParams parsePromqlParams(EsqlBaseParser.PromqlCommandContext ctx,
13181320
Instant start = null;
13191321
Instant end = null;
13201322
Duration step = null;
1323+
Integer buckets = null;
13211324
IndexPattern indexPattern = new IndexPattern(source, "*");
13221325

13231326
Set<String> paramsSeen = new HashSet<>();
@@ -1332,13 +1335,8 @@ private PromqlParams parsePromqlParams(EsqlBaseParser.PromqlCommandContext ctx,
13321335
case TIME -> time = PromqlParserUtils.parseDate(valueSource, parseParamValueString(paramCtx.value));
13331336
case START -> start = PromqlParserUtils.parseDate(valueSource, parseParamValueString(paramCtx.value));
13341337
case END -> end = PromqlParserUtils.parseDate(valueSource, parseParamValueString(paramCtx.value));
1335-
case STEP -> {
1336-
try {
1337-
step = Duration.ofSeconds(Integer.parseInt(parseParamValueString(paramCtx.value)));
1338-
} catch (NumberFormatException ignore) {
1339-
step = PromqlParserUtils.parseDuration(valueSource, parseParamValueString(paramCtx.value));
1340-
}
1341-
}
1338+
case STEP -> step = parsePositivePromqlDuration(valueSource, parseParamValueString(paramCtx.value), STEP);
1339+
case BUCKETS -> buckets = parsePositiveInteger(valueSource, parseParamValueString(paramCtx.value), BUCKETS);
13421340
case INDEX -> indexPattern = parseIndexPattern(paramCtx.value);
13431341
default -> {
13441342
String message = "Unknown parameter [{}]";
@@ -1353,19 +1351,22 @@ private PromqlParams parsePromqlParams(EsqlBaseParser.PromqlCommandContext ctx,
13531351

13541352
// Validation logic for time parameters
13551353
if (time != null) {
1356-
if (start != null || end != null || step != null) {
1354+
// instant query
1355+
if (step != null || buckets != null || start != null || end != null) {
13571356
throw new ParsingException(
13581357
source,
1359-
"Specify either [{}] for instant query or [{}], [{}] or [{}] for a range query",
1358+
"Specify either [{}] for instant query or any of [{}], [{}], [{}], [{}] for a range query",
13601359
TIME,
13611360
STEP,
1361+
BUCKETS,
13621362
START,
13631363
END
13641364
);
13651365
}
13661366
start = time;
13671367
end = time;
1368-
} else if (step != null || start != null || end != null) {
1368+
} else {
1369+
// range query
13691370
if (start != null || end != null) {
13701371
if (start == null || end == null) {
13711372
throw new ParsingException(
@@ -1384,21 +1385,31 @@ private PromqlParams parsePromqlParams(EsqlBaseParser.PromqlCommandContext ctx,
13841385
);
13851386
}
13861387
}
1387-
if (step == null) {
1388-
throw new ParsingException(source, "Parameter [{}] must be specified for a range query", STEP);
1389-
} else if (step.isPositive() == false) {
1390-
throw new ParsingException(
1391-
source,
1392-
"invalid parameter \"step\": zero or negative query resolution step widths are not accepted. "
1393-
+ "Try a positive integer",
1394-
step
1395-
);
1388+
if (step != null && buckets != null) {
1389+
throw new ParsingException(source, "Parameters [{}] and [{}] are mutually exclusive for a range query", STEP, BUCKETS);
1390+
}
1391+
if (step == null && buckets == null) {
1392+
buckets = DEFAULT_PROMQL_BUCKETS;
13961393
}
1397-
} else {
1398-
start = Instant.now();
1399-
end = start;
14001394
}
1401-
return new PromqlParams(source, start, end, step, indexPattern);
1395+
return new PromqlParams(source, start, end, step, buckets, indexPattern);
1396+
}
1397+
1398+
private Duration parsePositivePromqlDuration(Source source, String value, String parameterName) {
1399+
Duration parsedValue;
1400+
try {
1401+
parsedValue = Duration.ofSeconds(Integer.parseInt(value));
1402+
} catch (NumberFormatException ignore) {
1403+
try {
1404+
parsedValue = PromqlParserUtils.parseDuration(source, value);
1405+
} catch (ParsingException e) {
1406+
throw new ParsingException(source, "Invalid value [{}] for parameter [{}]", value, parameterName);
1407+
}
1408+
}
1409+
if (parsedValue.isPositive() == false) {
1410+
throw new ParsingException(source, "Invalid value [{}] for parameter [{}], expected a positive duration", value, parameterName);
1411+
}
1412+
return parsedValue;
14021413
}
14031414

14041415
private String parseParamName(EsqlBaseParser.PromqlParamNameContext ctx) {
@@ -1439,6 +1450,19 @@ private IndexPattern parseIndexPattern(EsqlBaseParser.PromqlParamValueContext ct
14391450
}
14401451
}
14411452

1453+
private Integer parsePositiveInteger(Source source, String value, String parameterName) {
1454+
int parsedValue;
1455+
try {
1456+
parsedValue = Integer.parseInt(value);
1457+
} catch (NumberFormatException e) {
1458+
throw new ParsingException(source, "Invalid value [{}] for parameter [{}], expected a positive integer", value, parameterName);
1459+
}
1460+
if (parsedValue <= 0) {
1461+
throw new ParsingException(source, "Invalid value [{}] for parameter [{}], expected a positive integer", value, parameterName);
1462+
}
1463+
return parsedValue;
1464+
}
1465+
14421466
public PlanFactory visitMmrCommand(EsqlBaseParser.MmrCommandContext ctx) {
14431467
Source source = source(ctx);
14441468

@@ -1485,7 +1509,7 @@ private Expression visitMMRQueryVector(EsqlBaseParser.MmrQueryVectorParamsContex
14851509
*
14861510
* @see <a href="https://prometheus.io/docs/prometheus/latest/querying/api/#expression-queries">PromQL API documentation</a>
14871511
*/
1488-
public record PromqlParams(Source source, Instant start, Instant end, Duration step, IndexPattern indexPattern) {
1512+
public record PromqlParams(Source source, Instant start, Instant end, Duration step, Integer buckets, IndexPattern indexPattern) {
14891513

14901514
public Literal startLiteral() {
14911515
if (start == null) {
@@ -1507,5 +1531,12 @@ public Literal stepLiteral() {
15071531
}
15081532
return Literal.timeDuration(source, step);
15091533
}
1534+
1535+
public Literal bucketsLiteral() {
1536+
if (buckets == null) {
1537+
return Literal.NULL;
1538+
}
1539+
return Literal.integer(source, buckets);
1540+
}
15101541
}
15111542
}

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/promql/PromqlCommand.java‎

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ public class PromqlCommand extends UnaryPlan implements TelemetryAware, PostAnal
5555
private final Literal start;
5656
private final Literal end;
5757
private final Literal step;
58+
private final Literal buckets;
5859
// TODO: this should be made available through the planner
5960
private final Expression timestamp;
6061
private final String valueColumnName;
@@ -70,10 +71,11 @@ public PromqlCommand(
7071
Literal start,
7172
Literal end,
7273
Literal step,
74+
Literal buckets,
7375
String valueColumnName,
7476
Expression timestamp
7577
) {
76-
this(source, child, promqlPlan, start, end, step, valueColumnName, new NameId(), new NameId(), timestamp);
78+
this(source, child, promqlPlan, start, end, step, buckets, valueColumnName, new NameId(), new NameId(), timestamp);
7779
}
7880

7981
// Range query constructor
@@ -84,6 +86,7 @@ public PromqlCommand(
8486
Literal start,
8587
Literal end,
8688
Literal step,
89+
Literal buckets,
8790
String valueColumnName,
8891
NameId valueId,
8992
NameId stepId,
@@ -94,6 +97,7 @@ public PromqlCommand(
9497
this.start = start;
9598
this.end = end;
9699
this.step = step;
100+
this.buckets = buckets;
97101
this.valueColumnName = valueColumnName;
98102
this.valueId = valueId;
99103
this.stepId = stepId;
@@ -110,6 +114,7 @@ protected NodeInfo<PromqlCommand> info() {
110114
start(),
111115
end(),
112116
step(),
117+
buckets(),
113118
valueColumnName(),
114119
valueId(),
115120
stepId(),
@@ -126,6 +131,7 @@ public PromqlCommand replaceChild(LogicalPlan newChild) {
126131
start(),
127132
end(),
128133
step(),
134+
buckets(),
129135
valueColumnName(),
130136
valueId(),
131137
stepId(),
@@ -141,6 +147,7 @@ public PromqlCommand withPromqlPlan(LogicalPlan newPromqlPlan) {
141147
start(),
142148
end(),
143149
step(),
150+
buckets(),
144151
valueColumnName(),
145152
valueId(),
146153
stepId(),
@@ -184,12 +191,16 @@ public Literal step() {
184191
return step;
185192
}
186193

194+
public Literal buckets() {
195+
return buckets;
196+
}
197+
187198
public boolean isInstantQuery() {
188-
return step.value() == null;
199+
return step.value() == null && buckets.value() == null;
189200
}
190201

191202
public boolean isRangeQuery() {
192-
return step.value() != null;
203+
return isInstantQuery() == false;
193204
}
194205

195206
public String valueColumnName() {
@@ -223,7 +234,7 @@ public List<Attribute> output() {
223234

224235
@Override
225236
public int hashCode() {
226-
return Objects.hash(child(), promqlPlan, start, end, step, valueColumnName, valueId, stepId, timestamp);
237+
return Objects.hash(child(), promqlPlan, start, end, step, buckets, valueColumnName, valueId, stepId, timestamp);
227238
}
228239

229240
@Override
@@ -236,6 +247,7 @@ public boolean equals(Object obj) {
236247
&& Objects.equals(start, other.start)
237248
&& Objects.equals(end, other.end)
238249
&& Objects.equals(step, other.step)
250+
&& Objects.equals(buckets, other.buckets)
239251
&& Objects.equals(valueColumnName, other.valueColumnName)
240252
&& Objects.equals(valueId, other.valueId)
241253
&& Objects.equals(stepId, other.stepId)
@@ -252,6 +264,7 @@ public String nodeString(NodeStringFormat format) {
252264
sb.append(" start=[").append(start);
253265
sb.append("] end=[").append(end);
254266
sb.append("] step=[").append(step);
267+
sb.append("] buckets=[").append(buckets);
255268
sb.append("] valueColumnName=[").append(valueColumnName);
256269
sb.append("] promql=[<>\n");
257270
sb.append(promqlPlan.toString());
@@ -270,6 +283,22 @@ protected AttributeSet computeReferences() {
270283
@Override
271284
public void postAnalysisVerification(Failures failures) {
272285
LogicalPlan p = promqlPlan();
286+
boolean hasStep = step.value() != null;
287+
boolean hasRangeAndBuckets = start.value() != null && end.value() != null && buckets.value() != null;
288+
if (hasStep == false && hasRangeAndBuckets == false) {
289+
failures.add(
290+
fail(
291+
this,
292+
"unable to create a bucket; provide either [{}] or all of [{}], [{}], and [{}] [{}]",
293+
"step",
294+
"start",
295+
"end",
296+
"buckets",
297+
sourceText()
298+
)
299+
);
300+
return;
301+
}
273302
// TODO(sidosera): Remove once instant query support is added.
274303
if (isInstantQuery()) {
275304
failures.add(fail(p, "instant queries are not supported at this time [{}]", sourceText()));

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/promql/PromqlVerifierTests.java‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,21 @@ public void testLogicalSetBinaryOperators() {
9595
public void testPromqlInstantQuery() {
9696
assertThat(
9797
error("PROMQL index=test time=\"2025-10-31T00:00:00Z\" (avg(foo))", tsdb),
98-
equalTo("1:48: instant queries are not supported at this time [PROMQL index=test time=\"2025-10-31T00:00:00Z\" (avg(foo))]")
98+
containsString("unable to create a bucket; provide either [step] or all of [start], [end], and [buckets]")
99+
);
100+
}
101+
102+
public void testPromqlMissingBucketParameters() {
103+
assertThat(
104+
error("PROMQL index=test avg(foo)", tsdb),
105+
containsString("unable to create a bucket; provide either [step] or all of [start], [end], and [buckets]")
106+
);
107+
}
108+
109+
public void testPromqlBucketsWithoutRange() {
110+
assertThat(
111+
error("PROMQL index=test buckets=10 avg(foo)", tsdb),
112+
containsString("unable to create a bucket; provide either [step] or all of [start], [end], and [buckets]")
99113
);
100114
}
101115

0 commit comments

Comments
 (0)