diff --git a/docs/changelog/126532.yaml b/docs/changelog/126532.yaml
new file mode 100644
index 0000000000000..dff3094c31ad8
--- /dev/null
+++ b/docs/changelog/126532.yaml
@@ -0,0 +1,6 @@
+pr: 126532
+summary: TO_IP can handle leading zeros
+area: ES|QL
+type: bug
+issues:
+ - 125460
diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/to_ip.md b/docs/reference/query-languages/esql/_snippets/functions/examples/to_ip.md
index 7f4256e24282e..f973ae8da4c24 100644
--- a/docs/reference/query-languages/esql/_snippets/functions/examples/to_ip.md
+++ b/docs/reference/query-languages/esql/_snippets/functions/examples/to_ip.md
@@ -1,6 +1,6 @@
% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
-**Example**
+**Examples**
```esql
ROW str1 = "1.1.1.1", str2 = "foo"
@@ -23,4 +23,26 @@ A following header will contain the failure reason and the offending value:
`"java.lang.IllegalArgumentException: 'foo' is not an IP string literal."`
+```esql
+ROW s = "1.1.010.1" | EVAL ip = TO_IP(s, {"leading_zeros":"octal"})
+```
+
+| s:keyword | ip:ip |
+| --- | --- |
+| 1.1.010.1 | 1.1.8.1 |
+
+
+Parse v4 addresses with leading zeros as octal. Like `ping` or `ftp`.
+
+```esql
+ROW s = "1.1.010.1" | EVAL ip = TO_IP(s, {"leading_zeros":"decimal"})
+```
+
+| s:keyword | ip:ip |
+| --- | --- |
+| 1.1.010.1 | 1.1.10.1 |
+
+
+Parse v4 addresses with leading zeros as decimal. Java's `InetAddress.getByName`.
+
diff --git a/docs/reference/query-languages/esql/_snippets/functions/functionNamedParams/to_ip.md b/docs/reference/query-languages/esql/_snippets/functions/functionNamedParams/to_ip.md
new file mode 100644
index 0000000000000..63705daf29798
--- /dev/null
+++ b/docs/reference/query-languages/esql/_snippets/functions/functionNamedParams/to_ip.md
@@ -0,0 +1,7 @@
+% This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../README.md for how to regenerate it.
+
+**Supported function named parameters**
+
+`leading_zeros`
+: (keyword) What to do with leading 0s in IPv4 addresses.
+
diff --git a/docs/reference/query-languages/esql/_snippets/functions/layout/to_ip.md b/docs/reference/query-languages/esql/_snippets/functions/layout/to_ip.md
index 1f62f7a098eb6..2ad1c59e09868 100644
--- a/docs/reference/query-languages/esql/_snippets/functions/layout/to_ip.md
+++ b/docs/reference/query-languages/esql/_snippets/functions/layout/to_ip.md
@@ -19,5 +19,8 @@
:::{include} ../types/to_ip.md
:::
+:::{include} ../functionNamedParams/to_ip.md
+:::
+
:::{include} ../examples/to_ip.md
:::
diff --git a/docs/reference/query-languages/esql/_snippets/functions/parameters/to_ip.md b/docs/reference/query-languages/esql/_snippets/functions/parameters/to_ip.md
index 6957a5264b050..7371ac6b8bb03 100644
--- a/docs/reference/query-languages/esql/_snippets/functions/parameters/to_ip.md
+++ b/docs/reference/query-languages/esql/_snippets/functions/parameters/to_ip.md
@@ -5,3 +5,6 @@
`field`
: Input value. The input can be a single- or multi-valued column or an expression.
+`options`
+: (Optional) Additional options.
+
diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/to_ip.md b/docs/reference/query-languages/esql/_snippets/functions/types/to_ip.md
index d8ffccaa6935a..e2d8481d8a370 100644
--- a/docs/reference/query-languages/esql/_snippets/functions/types/to_ip.md
+++ b/docs/reference/query-languages/esql/_snippets/functions/types/to_ip.md
@@ -2,9 +2,9 @@
**Supported types**
-| field | result |
-| --- | --- |
-| ip | ip |
-| keyword | ip |
-| text | ip |
+| field | options | result |
+| --- | --- | --- |
+| ip | | ip |
+| keyword | | ip |
+| text | | ip |
diff --git a/docs/reference/query-languages/esql/images/functions/to_ip.svg b/docs/reference/query-languages/esql/images/functions/to_ip.svg
index 47cd11f4f6ed2..a5127605ffc1b 100644
--- a/docs/reference/query-languages/esql/images/functions/to_ip.svg
+++ b/docs/reference/query-languages/esql/images/functions/to_ip.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/to_ip.json b/docs/reference/query-languages/esql/kibana/definition/functions/to_ip.json
index 50fbcd339e26f..7ed43181d3105 100644
--- a/docs/reference/query-languages/esql/kibana/definition/functions/to_ip.json
+++ b/docs/reference/query-languages/esql/kibana/definition/functions/to_ip.json
@@ -42,7 +42,9 @@
}
],
"examples" : [
- "ROW str1 = \"1.1.1.1\", str2 = \"foo\"\n| EVAL ip1 = TO_IP(str1), ip2 = TO_IP(str2)\n| WHERE CIDR_MATCH(ip1, \"1.0.0.0/8\")"
+ "ROW str1 = \"1.1.1.1\", str2 = \"foo\"\n| EVAL ip1 = TO_IP(str1), ip2 = TO_IP(str2)\n| WHERE CIDR_MATCH(ip1, \"1.0.0.0/8\")",
+ "ROW s = \"1.1.010.1\" | EVAL ip = TO_IP(s, {\"leading_zeros\":\"octal\"})",
+ "ROW s = \"1.1.010.1\" | EVAL ip = TO_IP(s, {\"leading_zeros\":\"decimal\"})"
],
"preview" : false,
"snapshot_only" : false
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java
index 9575f8ee176ea..49dd99346b932 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java
@@ -133,6 +133,7 @@ public class CsvTestsDataLoader {
private static final TestDataset ADDRESSES = new TestDataset("addresses");
private static final TestDataset BOOKS = new TestDataset("books").withSetting("books-settings.json");
private static final TestDataset SEMANTIC_TEXT = new TestDataset("semantic_text").withInferenceEndpoint(true);
+ private static final TestDataset LOGS = new TestDataset("logs");
public static final Map CSV_DATASET_MAP = Map.ofEntries(
Map.entry(EMPLOYEES.indexName, EMPLOYEES),
@@ -182,7 +183,8 @@ public class CsvTestsDataLoader {
Map.entry(DISTANCES.indexName, DISTANCES),
Map.entry(ADDRESSES.indexName, ADDRESSES),
Map.entry(BOOKS.indexName, BOOKS),
- Map.entry(SEMANTIC_TEXT.indexName, SEMANTIC_TEXT)
+ Map.entry(SEMANTIC_TEXT.indexName, SEMANTIC_TEXT),
+ Map.entry(LOGS.indexName, LOGS)
);
private static final EnrichConfig LANGUAGES_ENRICH = new EnrichConfig("languages_policy", "enrich-policy-languages.json");
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/logs.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/logs.csv
new file mode 100644
index 0000000000000..2fd232a3ed33d
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/logs.csv
@@ -0,0 +1,5 @@
+@timestamp:date ,system:keyword,message:keyword
+2023-10-23T13:55:01.543Z, ping,Pinging 192.168.86.046
+2023-10-23T13:55:01.544Z, cron,Running cats
+2023-10-23T13:55:01.545Z, java,Doing java stuff for 192.168.86.038
+2023-10-23T13:55:01.546Z, java,More java stuff
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec
index e8d59628c29fc..50c46beeff327 100644
--- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/ip.csv-spec
@@ -271,6 +271,86 @@ str1:keyword |str2:keyword |ip1:ip |ip2:ip
// end::to_ip-result[]
;
+convertFromStringLeadingZerosDecimal
+required_capability: to_ip_leading_zeros
+// tag::to_ip_leading_zeros_decimal[]
+ROW s = "1.1.010.1" | EVAL ip = TO_IP(s, {"leading_zeros":"decimal"})
+// end::to_ip_leading_zeros_decimal[]
+;
+
+// tag::to_ip_leading_zeros_decimal-result[]
+s:keyword | ip:ip
+1.1.010.1 | 1.1.10.1
+// end::to_ip_leading_zeros_decimal-result[]
+;
+
+convertFromStringLeadingZerosOctal
+required_capability: to_ip_leading_zeros
+// tag::to_ip_leading_zeros_octal[]
+ROW s = "1.1.010.1" | EVAL ip = TO_IP(s, {"leading_zeros":"octal"})
+// end::to_ip_leading_zeros_octal[]
+;
+
+// tag::to_ip_leading_zeros_octal-result[]
+s:keyword | ip:ip
+1.1.010.1 | 1.1.8.1
+// end::to_ip_leading_zeros_octal-result[]
+;
+
+convertFromStringFancy
+required_capability: to_ip_leading_zeros
+FROM logs
+| KEEP @timestamp, system, message
+| EVAL client = CASE(
+ system == "ping",
+ TO_IP(REPLACE(message, "Pinging ", ""), {"leading_zeros": "octal"}),
+ system == "java" AND STARTS_WITH(message, "Doing java stuff for "),
+ TO_IP(REPLACE(message, "Doing java stuff for ", ""), {"leading_zeros": "decimal"}))
+| SORT @timestamp
+| LIMIT 4
+;
+
+@timestamp:date |system:keyword|message:keyword |client:ip
+2023-10-23T13:55:01.543Z| ping|Pinging 192.168.86.046 |192.168.86.38
+2023-10-23T13:55:01.544Z| cron|Running cats |null
+2023-10-23T13:55:01.545Z| java|Doing java stuff for 192.168.86.038|192.168.86.38
+2023-10-23T13:55:01.546Z| java|More java stuff |null
+;
+
+toIpInAgg
+ROW s = "1.1.1.1" | STATS COUNT(*) BY ip = TO_IP(s)
+;
+
+COUNT(*):long | ip:ip
+ 1 | 1.1.1.1
+;
+
+toIpInSort
+ROW s = "1.1.1.1" | SORT TO_IP(s)
+;
+
+s:keyword
+1.1.1.1
+;
+
+toIpInAggOctal
+required_capability: to_ip_leading_zeros
+ROW s = "1.1.010.1" | STATS COUNT(*) BY ip = TO_IP(s, {"leading_zeros":"octal"})
+;
+
+COUNT(*):long | ip:ip
+ 1 | 1.1.8.1
+;
+
+toIpInSortOctal
+required_capability: to_ip_leading_zeros
+ROW s = "1.1.010.1" | SORT TO_IP(s, {"leading_zeros":"octal"})
+;
+
+s:keyword
+1.1.010.1
+;
+
cdirMatchOrsIPs
required_capability: combine_disjunctive_cidrmatches
diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-logs.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-logs.json
new file mode 100644
index 0000000000000..8b1ddb0e299db
--- /dev/null
+++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-logs.json
@@ -0,0 +1,13 @@
+{
+ "properties" : {
+ "@timestamp" : {
+ "type" : "date"
+ },
+ "system" : {
+ "type" : "keyword"
+ },
+ "message" : {
+ "type" : "keyword"
+ }
+ }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
index 406a7f5f6f08d..45b2d45483858 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
@@ -982,6 +982,11 @@ public enum Cap {
*/
FORK_V2(Build.current().isSnapshot()),
+ /**
+ * Support for the {@code leading_zeros} named parameter.
+ */
+ TO_IP_LEADING_ZEROS,
+
/**
* Does the usage information for ESQL contain a histogram of {@code took} values?
*/
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
index 980350ce43d4e..b87b7fdc58408 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java
@@ -58,6 +58,7 @@
import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Greatest;
import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Least;
import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction;
+import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ConvertFunction;
import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FoldablesConvertFunction;
import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDouble;
import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToInteger;
@@ -70,6 +71,7 @@
import org.elasticsearch.xpack.esql.index.EsIndex;
import org.elasticsearch.xpack.esql.index.IndexResolution;
import org.elasticsearch.xpack.esql.inference.ResolvedInference;
+import org.elasticsearch.xpack.esql.optimizer.rules.logical.SubstituteSurrogateExpressions;
import org.elasticsearch.xpack.esql.parser.ParsingException;
import org.elasticsearch.xpack.esql.plan.IndexPattern;
import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
@@ -1578,10 +1580,12 @@ private LogicalPlan doRule(LogicalPlan plan) {
int alreadyAddedUnionFieldAttributes = unionFieldAttributes.size();
// See if the eval function has an unresolved MultiTypeEsField field
// Replace the entire convert function with a new FieldAttribute (containing type conversion knowledge)
- plan = plan.transformExpressionsOnly(
- AbstractConvertFunction.class,
- convert -> resolveConvertFunction(convert, unionFieldAttributes)
- );
+ plan = plan.transformExpressionsOnly(e -> {
+ if (e instanceof ConvertFunction convert) {
+ return resolveConvertFunction(convert, unionFieldAttributes);
+ }
+ return e;
+ });
// If no union fields were generated, return the plan as is
if (unionFieldAttributes.size() == alreadyAddedUnionFieldAttributes) {
return plan;
@@ -1612,7 +1616,8 @@ private LogicalPlan doRule(LogicalPlan plan) {
return plan;
}
- private Expression resolveConvertFunction(AbstractConvertFunction convert, List unionFieldAttributes) {
+ private Expression resolveConvertFunction(ConvertFunction convert, List unionFieldAttributes) {
+ Expression convertExpression = (Expression) convert;
if (convert.field() instanceof FieldAttribute fa && fa.field() instanceof InvalidMappedField imf) {
HashMap typeResolutions = new HashMap<>();
Set supportedTypes = convert.supportedTypes();
@@ -1639,9 +1644,11 @@ private Expression resolveConvertFunction(AbstractConvertFunction convert, List<
return createIfDoesNotAlreadyExist(fa, resolvedField, unionFieldAttributes);
}
} else if (convert.field() instanceof AbstractConvertFunction subConvert) {
- return convert.replaceChildren(Collections.singletonList(resolveConvertFunction(subConvert, unionFieldAttributes)));
+ return convertExpression.replaceChildren(
+ Collections.singletonList(resolveConvertFunction(subConvert, unionFieldAttributes))
+ );
}
- return convert;
+ return convertExpression;
}
private Expression createIfDoesNotAlreadyExist(
@@ -1677,7 +1684,7 @@ private MultiTypeEsField resolvedMultiTypeEsField(FieldAttribute fa, HashMap unaryScalars() {
entries.add(ToGeoShape.ENTRY);
entries.add(ToCartesianShape.ENTRY);
entries.add(ToGeoPoint.ENTRY);
- entries.add(ToIP.ENTRY);
+ entries.add(ToIpLeadingZerosDecimal.ENTRY);
+ entries.add(ToIpLeadingZerosOctal.ENTRY);
+ entries.add(ToIpLeadingZerosRejected.ENTRY);
entries.add(ToInteger.ENTRY);
entries.add(ToLong.ENTRY);
entries.add(ToRadians.ENTRY);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
index 764f83a67e925..cda2d5593b745 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
@@ -16,6 +16,7 @@
import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.core.util.Check;
+import org.elasticsearch.xpack.esql.expression.SurrogateExpression;
import org.elasticsearch.xpack.esql.expression.function.aggregate.Avg;
import org.elasticsearch.xpack.esql.expression.function.aggregate.AvgOverTime;
import org.elasticsearch.xpack.esql.expression.function.aggregate.Count;
@@ -56,8 +57,11 @@
import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDouble;
import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToGeoPoint;
import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToGeoShape;
-import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToIP;
import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToInteger;
+import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToIp;
+import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToIpLeadingZerosDecimal;
+import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToIpLeadingZerosOctal;
+import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToIpLeadingZerosRejected;
import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToLong;
import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToRadians;
import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToString;
@@ -230,6 +234,7 @@ public class EsqlFunctionRegistry {
public EsqlFunctionRegistry() {
register(functions());
buildDataTypesForStringLiteralConversion(functions());
+ nameSurrogates();
}
EsqlFunctionRegistry(FunctionDefinition... functions) {
@@ -391,7 +396,7 @@ private static FunctionDefinition[][] functions() {
def(ToDouble.class, ToDouble::new, "to_double", "to_dbl"),
def(ToGeoPoint.class, ToGeoPoint::new, "to_geopoint"),
def(ToGeoShape.class, ToGeoShape::new, "to_geoshape"),
- def(ToIP.class, ToIP::new, "to_ip"),
+ def(ToIp.class, ToIp::new, "to_ip"),
def(ToInteger.class, ToInteger::new, "to_integer", "to_int"),
def(ToLong.class, ToLong::new, "to_long"),
def(ToRadians.class, ToRadians::new, "to_radians"),
@@ -795,6 +800,15 @@ protected void buildDataTypesForStringLiteralConversion(FunctionDefinition[]...
}
}
+ /**
+ * Add {@link #names} entries for functions that are not registered, but we rewrite to using {@link SurrogateExpression}.
+ */
+ private void nameSurrogates() {
+ names.put(ToIpLeadingZerosRejected.class, "TO_IP");
+ names.put(ToIpLeadingZerosDecimal.class, "TO_IP");
+ names.put(ToIpLeadingZerosOctal.class, "TO_IP");
+ }
+
protected interface FunctionBuilder {
Function build(Source source, List children, Configuration cfg);
}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java
index 5e10e8984f56e..3b869c0200cb9 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java
@@ -43,7 +43,7 @@
* {@link org.elasticsearch.xpack.esql.expression.function.scalar}.
*
*/
-public abstract class AbstractConvertFunction extends UnaryScalarFunction {
+public abstract class AbstractConvertFunction extends UnaryScalarFunction implements ConvertFunction {
// the numeric types convert functions need to handle; the other numeric types are converted upstream to one of these
private static final List NUMERIC_TYPES = List.of(DataType.INTEGER, DataType.LONG, DataType.UNSIGNED_LONG, DataType.DOUBLE);
@@ -76,11 +76,12 @@ protected TypeResolution resolveType() {
return isTypeOrUnionType(field(), factories()::containsKey, sourceText(), null, supportedTypesNames(supportedTypes()));
}
+ @Override
public Set supportedTypes() {
return factories().keySet();
}
- private static String supportedTypesNames(Set types) {
+ static String supportedTypesNames(Set types) {
List supportedTypesNames = new ArrayList<>(types.size());
HashSet supportTypes = new HashSet<>(types);
if (supportTypes.containsAll(NUMERIC_TYPES)) {
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ConvertFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ConvertFunction.java
new file mode 100644
index 0000000000000..2969e4c154403
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ConvertFunction.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+
+import java.util.Set;
+
+/**
+ * A function that converts from one type to another.
+ */
+public interface ConvertFunction {
+ /**
+ * Expression containing the values to be converted.
+ */
+ Expression field();
+
+ /**
+ * The types that {@link #field()} can have.
+ */
+ Set supportedTypes();
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ParseIp.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ParseIp.java
index e83c85614f24a..cba986f7e4533 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ParseIp.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ParseIp.java
@@ -42,6 +42,14 @@ public class ParseIp {
return new ParseIpLeadingZerosRejectedEvaluator.Factory(source, field, driverContext -> buildScratch(driverContext.breaker()));
};
+ static final AbstractConvertFunction.BuildFactory FROM_KEYWORD_LEADING_ZEROS_DECIMAL = (source, field) -> {
+ return new ParseIpLeadingZerosAreDecimalEvaluator.Factory(source, field, driverContext -> buildScratch(driverContext.breaker()));
+ };
+
+ static final AbstractConvertFunction.BuildFactory FROM_KEYWORD_LEADING_ZEROS_OCTAL = (source, field) -> {
+ return new ParseIpLeadingZerosAreOctalEvaluator.Factory(source, field, driverContext -> buildScratch(driverContext.breaker()));
+ };
+
public static BreakingBytesRefBuilder buildScratch(CircuitBreaker breaker) {
BreakingBytesRefBuilder scratch = new BreakingBytesRefBuilder(breaker, "to_ip", 16);
scratch.setLength(InetAddressPoint.BYTES);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIp.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIp.java
new file mode 100644
index 0000000000000..dd25015a6e6e3
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIp.java
@@ -0,0 +1,255 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.compute.operator.EvalOperator;
+import org.elasticsearch.xpack.esql.core.expression.EntryExpression;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.Literal;
+import org.elasticsearch.xpack.esql.core.expression.MapExpression;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.expression.SurrogateExpression;
+import org.elasticsearch.xpack.esql.expression.function.Example;
+import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
+import org.elasticsearch.xpack.esql.expression.function.MapParam;
+import org.elasticsearch.xpack.esql.expression.function.OptionalArgument;
+import org.elasticsearch.xpack.esql.expression.function.Param;
+import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isMapExpression;
+import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isTypeOrUnionType;
+import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction.supportedTypesNames;
+
+/**
+ * Converts strings to IPs.
+ *
+ * IPv4 addresses have traditionally parsed quads with leading zeros in three
+ * mutually exclusive ways:
+ *
+ *
+ *
As octal numbers. So {@code 1.1.010.1} becomes {@code 1.1.8.1}.
+ *
As decimal numbers. So {@code 1.1.010.1} becomes {@code 1.1.10.1}.
+ *
Rejects them entirely. So {@code 1.1.010.1} becomes {@code null} with a warning.
+ *
+ *
+ * These three ways of handling leading zeros are available with the optional
+ * {@code leading_zeros} named parameter. Set to {@code octal}, {@code decimal},
+ * or {@code reject}. If not sent this defaults to {@code reject} which has
+ * been Elasticsearch's traditional way of handling leading zeros for years.
+ *
+ *
+ * This doesn't extend from {@link AbstractConvertFunction} so that it can
+ * support a named parameter for the leading zeros behavior. Instead, it rewrites
+ * itself into either {@link ToIpLeadingZerosOctal}, {@link ToIpLeadingZerosDecimal},
+ * or {@link ToIpLeadingZerosRejected} which are all {@link AbstractConvertFunction}
+ * subclasses. This keeps the conversion code happy while still allowing us to
+ * expose a single method to users.
+ *
+ */
+public class ToIp extends EsqlScalarFunction implements SurrogateExpression, OptionalArgument, ConvertFunction {
+ private static final String LEADING_ZEROS = "leading_zeros";
+ public static final Map ALLOWED_OPTIONS = Map.ofEntries(Map.entry(LEADING_ZEROS, KEYWORD));
+
+ private final Expression field;
+ private final Expression options;
+
+ @FunctionInfo(
+ returnType = "ip",
+ description = "Converts an input string to an IP value.",
+ examples = {
+ @Example(file = "ip", tag = "to_ip", explanation = """
+ Note that in this example, the last conversion of the string isn’t possible.
+ When this happens, the result is a `null` value. In this case a _Warning_ header is added to the response.
+ The header will provide information on the source of the failure:
+
+ `"Line 1:68: evaluation of [TO_IP(str2)] failed, treating result as null. Only first 20 failures recorded."`
+
+ A following header will contain the failure reason and the offending value:
+
+ `"java.lang.IllegalArgumentException: 'foo' is not an IP string literal."`"""),
+ @Example(file = "ip", tag = "to_ip_leading_zeros_octal", explanation = """
+ Parse v4 addresses with leading zeros as octal. Like `ping` or `ftp`.
+ """),
+ @Example(file = "ip", tag = "to_ip_leading_zeros_decimal", explanation = """
+ Parse v4 addresses with leading zeros as decimal. Java's `InetAddress.getByName`.
+ """) }
+ )
+ public ToIp(
+ Source source,
+ @Param(
+ name = "field",
+ type = { "ip", "keyword", "text" },
+ description = "Input value. The input can be a single- or multi-valued column or an expression."
+ ) Expression field,
+ @MapParam(
+ name = "options",
+ params = {
+ @MapParam.MapParamEntry(
+ name = "leading_zeros",
+ type = "keyword",
+ valueHint = { "reject", "octal", "decimal" },
+ description = "What to do with leading 0s in IPv4 addresses."
+ ) },
+ description = "(Optional) Additional options.",
+ optional = true
+ ) Expression options
+ ) {
+ super(source, options == null ? List.of(field) : List.of(field, options));
+ this.field = field;
+ this.options = options;
+ }
+
+ @Override
+ public String getWriteableName() {
+ throw new UnsupportedOperationException("not serialized");
+ }
+
+ @Override
+ public void writeTo(StreamOutput out) throws IOException {
+ throw new UnsupportedOperationException("not serialized");
+ }
+
+ @Override
+ public DataType dataType() {
+ return IP;
+ }
+
+ @Override
+ public Expression replaceChildren(List newChildren) {
+ return new ToIp(source(), newChildren.get(0), newChildren.size() == 1 ? null : newChildren.get(1));
+ }
+
+ @Override
+ protected NodeInfo extends Expression> info() {
+ return NodeInfo.create(this, ToIp::new, field, options);
+ }
+
+ @Override
+ public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) {
+ throw new UnsupportedOperationException("should be rewritten");
+ }
+
+ @Override
+ public Expression surrogate() {
+ return LeadingZeros.from((MapExpression) options).surrogate(source(), field);
+ }
+
+ @Override
+ public Expression field() {
+ return field;
+ }
+
+ @Override
+ public Set supportedTypes() {
+ // All ToIpLeadingZeros* functions support the same input set. So we just pick one.
+ return ToIpLeadingZerosRejected.EVALUATORS.keySet();
+ }
+
+ @Override
+ protected TypeResolution resolveType() {
+ if (childrenResolved() == false) {
+ return new TypeResolution("Unresolved children");
+ }
+ TypeResolution resolution = isTypeOrUnionType(
+ field,
+ ToIpLeadingZerosRejected.EVALUATORS::containsKey,
+ sourceText(),
+ null,
+ supportedTypesNames(supportedTypes())
+ );
+ if (resolution.unresolved()) {
+ return resolution;
+ }
+ if (options == null) {
+ return resolution;
+ }
+ resolution = isMapExpression(options, sourceText(), SECOND);
+ if (resolution.unresolved()) {
+ return resolution;
+ }
+ for (EntryExpression e : ((MapExpression) options).entryExpressions()) {
+ String key;
+ if (e.key().dataType() != KEYWORD) {
+ return new TypeResolution("map keys must be strings");
+ }
+ if (e.key() instanceof Literal keyl) {
+ key = (String) keyl.value();
+ } else {
+ return new TypeResolution("map keys must be literals");
+ }
+ DataType expected = ALLOWED_OPTIONS.get(key);
+ if (expected == null) {
+ return new TypeResolution("[" + key + "] is not a supported option");
+ }
+
+ if (e.value().dataType() != expected) {
+ return new TypeResolution("[" + key + "] expects [" + expected + "] but was [" + e.value().dataType() + "]");
+ }
+ if (e.value() instanceof Literal == false) {
+ return new TypeResolution("map values must be literals");
+ }
+ }
+ try {
+ LeadingZeros.from((MapExpression) options);
+ } catch (IllegalArgumentException e) {
+ return new TypeResolution(e.getMessage());
+ }
+ return TypeResolution.TYPE_RESOLVED;
+ }
+
+ public enum LeadingZeros {
+ REJECT {
+ @Override
+ public Expression surrogate(Source source, Expression field) {
+ return new ToIpLeadingZerosRejected(source, field);
+ }
+ },
+ DECIMAL {
+ @Override
+ public Expression surrogate(Source source, Expression field) {
+ return new ToIpLeadingZerosDecimal(source, field);
+ }
+ },
+ OCTAL {
+ @Override
+ public Expression surrogate(Source source, Expression field) {
+ return new ToIpLeadingZerosOctal(source, field);
+ }
+ };
+
+ public static LeadingZeros from(MapExpression exp) {
+ if (exp == null) {
+ return REJECT;
+ }
+ Expression e = exp.keyFoldedMap().get(LEADING_ZEROS);
+ return e == null ? REJECT : from((String) ((Literal) e).value());
+ }
+
+ public static LeadingZeros from(String str) {
+ return switch (str) {
+ case "reject" -> REJECT;
+ case "octal" -> OCTAL;
+ case "decimal" -> DECIMAL;
+ default -> throw new IllegalArgumentException("Illegal leading_zeros [" + str + "]");
+ };
+ }
+
+ public abstract Expression surrogate(Source source, Expression field);
+ }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIpLeadingZerosDecimal.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIpLeadingZerosDecimal.java
new file mode 100644
index 0000000000000..2754dcf39ff73
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIpLeadingZerosDecimal.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+import static org.elasticsearch.xpack.esql.expression.function.scalar.convert.ParseIp.FROM_KEYWORD_LEADING_ZEROS_DECIMAL;
+
+public class ToIpLeadingZerosDecimal extends AbstractConvertFunction {
+ public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+ Expression.class,
+ "ToIpLeadingZerosDecimal",
+ ToIpLeadingZerosDecimal::new
+ );
+
+ static final Map EVALUATORS = Map.ofEntries(
+ Map.entry(IP, (source, field) -> field),
+ Map.entry(KEYWORD, FROM_KEYWORD_LEADING_ZEROS_DECIMAL),
+ Map.entry(TEXT, FROM_KEYWORD_LEADING_ZEROS_DECIMAL)
+ );
+
+ public ToIpLeadingZerosDecimal(Source source, Expression field) {
+ super(source, field);
+ }
+
+ private ToIpLeadingZerosDecimal(StreamInput in) throws IOException {
+ super(in);
+ }
+
+ @Override
+ public String getWriteableName() {
+ return ENTRY.name;
+ }
+
+ @Override
+ protected Map factories() {
+ return EVALUATORS;
+ }
+
+ @Override
+ public DataType dataType() {
+ return IP;
+ }
+
+ @Override
+ public Expression replaceChildren(List newChildren) {
+ return new ToIpLeadingZerosDecimal(source(), newChildren.get(0));
+ }
+
+ @Override
+ protected NodeInfo extends Expression> info() {
+ return NodeInfo.create(this, ToIpLeadingZerosDecimal::new, field());
+ }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIpLeadingZerosOctal.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIpLeadingZerosOctal.java
new file mode 100644
index 0000000000000..404b6c27519c8
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIpLeadingZerosOctal.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
+
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
+import org.elasticsearch.xpack.esql.core.tree.Source;
+import org.elasticsearch.xpack.esql.core.type.DataType;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
+import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
+import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
+import static org.elasticsearch.xpack.esql.expression.function.scalar.convert.ParseIp.FROM_KEYWORD_LEADING_ZEROS_OCTAL;
+
+public class ToIpLeadingZerosOctal extends AbstractConvertFunction {
+ public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+ Expression.class,
+ "ToIpLeadingZerosOctal",
+ ToIpLeadingZerosOctal::new
+ );
+
+ static final Map EVALUATORS = Map.ofEntries(
+ Map.entry(IP, (source, field) -> field),
+ Map.entry(KEYWORD, FROM_KEYWORD_LEADING_ZEROS_OCTAL),
+ Map.entry(TEXT, FROM_KEYWORD_LEADING_ZEROS_OCTAL)
+ );
+
+ public ToIpLeadingZerosOctal(Source source, Expression field) {
+ super(source, field);
+ }
+
+ private ToIpLeadingZerosOctal(StreamInput in) throws IOException {
+ super(in);
+ }
+
+ @Override
+ public String getWriteableName() {
+ return ENTRY.name;
+ }
+
+ @Override
+ protected Map factories() {
+ return EVALUATORS;
+ }
+
+ @Override
+ public DataType dataType() {
+ return IP;
+ }
+
+ @Override
+ public Expression replaceChildren(List newChildren) {
+ return new ToIpLeadingZerosOctal(source(), newChildren.get(0));
+ }
+
+ @Override
+ protected NodeInfo extends Expression> info() {
+ return NodeInfo.create(this, ToIpLeadingZerosOctal::new, field());
+ }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIP.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIpLeadingZerosRejected.java
similarity index 52%
rename from x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIP.java
rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIpLeadingZerosRejected.java
index 77f26269b1c79..a02f08b50c045 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIP.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIpLeadingZerosRejected.java
@@ -13,9 +13,6 @@
import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.expression.function.Example;
-import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
-import org.elasticsearch.xpack.esql.expression.function.Param;
import java.io.IOException;
import java.util.List;
@@ -26,41 +23,28 @@
import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT;
import static org.elasticsearch.xpack.esql.expression.function.scalar.convert.ParseIp.FROM_KEYWORD_LEADING_ZEROS_REJECTED;
-public class ToIP extends AbstractConvertFunction {
- public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "ToIP", ToIP::new);
+public class ToIpLeadingZerosRejected extends AbstractConvertFunction {
+ public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(
+ Expression.class,
+ /*
+ * This is the name a function with this behavior has had since the
+ * dawn of ESQL. The ToIp function that exists now is not serialized.
+ */
+ "ToIP",
+ ToIpLeadingZerosRejected::new
+ );
- private static final Map EVALUATORS = Map.ofEntries(
+ static final Map EVALUATORS = Map.ofEntries(
Map.entry(IP, (source, field) -> field),
Map.entry(KEYWORD, FROM_KEYWORD_LEADING_ZEROS_REJECTED),
Map.entry(TEXT, FROM_KEYWORD_LEADING_ZEROS_REJECTED)
);
- @FunctionInfo(
- returnType = "ip",
- description = "Converts an input string to an IP value.",
- examples = @Example(file = "ip", tag = "to_ip", explanation = """
- Note that in this example, the last conversion of the string isn’t possible.
- When this happens, the result is a `null` value. In this case a _Warning_ header is added to the response.
- The header will provide information on the source of the failure:
-
- `"Line 1:68: evaluation of [TO_IP(str2)] failed, treating result as null. Only first 20 failures recorded."`
-
- A following header will contain the failure reason and the offending value:
-
- `"java.lang.IllegalArgumentException: 'foo' is not an IP string literal."`""")
- )
- public ToIP(
- Source source,
- @Param(
- name = "field",
- type = { "ip", "keyword", "text" },
- description = "Input value. The input can be a single- or multi-valued column or an expression."
- ) Expression field
- ) {
+ public ToIpLeadingZerosRejected(Source source, Expression field) {
super(source, field);
}
- private ToIP(StreamInput in) throws IOException {
+ private ToIpLeadingZerosRejected(StreamInput in) throws IOException {
super(in);
}
@@ -81,11 +65,11 @@ public DataType dataType() {
@Override
public Expression replaceChildren(List newChildren) {
- return new ToIP(source(), newChildren.get(0));
+ return new ToIpLeadingZerosRejected(source(), newChildren.get(0));
}
@Override
protected NodeInfo extends Expression> info() {
- return NodeInfo.create(this, ToIP::new, field());
+ return NodeInfo.create(this, ToIpLeadingZerosRejected::new, field());
}
}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java
index 295116a5e99c2..0316f56b297da 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesFunction.java
@@ -34,6 +34,7 @@
import org.elasticsearch.xpack.esql.core.util.Check;
import org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes;
import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper;
+import org.elasticsearch.xpack.esql.expression.SurrogateExpression;
import org.elasticsearch.xpack.esql.optimizer.rules.physical.local.LucenePushdownPredicates;
import org.elasticsearch.xpack.esql.planner.TranslatorHandler;
import org.elasticsearch.xpack.esql.querydsl.query.SpatialRelatesQuery;
@@ -48,7 +49,8 @@ public abstract class SpatialRelatesFunction extends BinarySpatialFunction
implements
EvaluatorMapper,
SpatialEvaluatorFactory.SpatialSourceSupplier,
- TranslationAware {
+ TranslationAware,
+ SurrogateExpression {
protected SpatialRelatesFunction(Source source, Expression left, Expression right, boolean leftDocValues, boolean rightDocValues) {
super(source, left, right, leftDocValues, rightDocValues, false);
@@ -73,6 +75,7 @@ public DataType dataType() {
/**
* Some spatial functions can replace themselves with alternatives that are more efficient for certain cases.
*/
+ @Override
public SpatialRelatesFunction surrogate() {
return this;
}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
index a166e81ec688f..ffa0f2bfec73f 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java
@@ -58,9 +58,9 @@
import org.elasticsearch.xpack.esql.optimizer.rules.logical.SkipQueryOnLimitZero;
import org.elasticsearch.xpack.esql.optimizer.rules.logical.SplitInWithFoldableValue;
import org.elasticsearch.xpack.esql.optimizer.rules.logical.SubstituteFilteredExpression;
-import org.elasticsearch.xpack.esql.optimizer.rules.logical.SubstituteSpatialSurrogates;
+import org.elasticsearch.xpack.esql.optimizer.rules.logical.SubstituteSurrogateAggregations;
+import org.elasticsearch.xpack.esql.optimizer.rules.logical.SubstituteSurrogateExpressions;
import org.elasticsearch.xpack.esql.optimizer.rules.logical.SubstituteSurrogatePlans;
-import org.elasticsearch.xpack.esql.optimizer.rules.logical.SubstituteSurrogates;
import org.elasticsearch.xpack.esql.optimizer.rules.logical.TranslateTimeSeriesAggregate;
import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
import org.elasticsearch.xpack.esql.rule.ParameterizedRuleExecutor;
@@ -137,11 +137,12 @@ protected static Batch substitutions() {
// then extract nested aggs top-level
new ReplaceAggregateAggExpressionWithEval(),
// lastly replace surrogate functions
- new SubstituteSurrogates(),
+ new SubstituteSurrogateAggregations(),
+ // translate metric aggregates after surrogate substitution and replace nested expressions with eval (again)
new TranslateTimeSeriesAggregate(),
new PruneUnusedIndexMode(),
// after translating metric aggregates, we need to replace surrogate substitutions and nested expressions again.
- new SubstituteSurrogates(),
+ new SubstituteSurrogateAggregations(),
new ReplaceAggregateNestedExpressionWithEval(),
// this one needs to be placed before ReplaceAliasingEvalWithProject, so that any potential aliasing eval (eval x = y)
// is not replaced with a Project before the eval to be copied on the left hand side of an InlineJoin
@@ -150,7 +151,7 @@ protected static Batch substitutions() {
new ReplaceTrivialTypeConversions(),
new ReplaceAliasingEvalWithProject(),
new SkipQueryOnEmptyMappings(),
- new SubstituteSpatialSurrogates(),
+ new SubstituteSurrogateExpressions(),
new ReplaceOrderByExpressionWithEval()
// new NormalizeAggregate(), - waits on https://github.com/elastic/elasticsearch/issues/100634
);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteSpatialSurrogates.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteSpatialSurrogates.java
deleted file mode 100644
index 4b68ee941bc92..0000000000000
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteSpatialSurrogates.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.esql.optimizer.rules.logical;
-
-import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction;
-import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
-
-/**
- * Currently this works similarly to SurrogateExpression, leaving the logic inside the expressions,
- * so each can decide for itself whether or not to change to a surrogate expression.
- * But what is actually being done is similar to LiteralsOnTheRight. We can consider in the future moving
- * this in either direction, reducing the number of rules, but for now,
- * it's a separate rule to reduce the risk of unintended interactions with other rules.
- */
-public final class SubstituteSpatialSurrogates extends OptimizerRules.OptimizerExpressionRule {
-
- public SubstituteSpatialSurrogates() {
- super(OptimizerRules.TransformDirection.UP);
- }
-
- @Override
- protected SpatialRelatesFunction rule(SpatialRelatesFunction function, LogicalOptimizerContext ctx) {
- return function.surrogate();
- }
-}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteSurrogates.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteSurrogateAggregations.java
similarity index 97%
rename from x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteSurrogates.java
rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteSurrogateAggregations.java
index f667e7c97e4f7..c762015dc597e 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteSurrogates.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteSurrogateAggregations.java
@@ -32,10 +32,8 @@
import java.util.List;
import java.util.Map;
-public final class SubstituteSurrogates extends OptimizerRules.OptimizerRule {
- // TODO: currently this rule only works for aggregate functions (AVG)
-
- public SubstituteSurrogates() {
+public final class SubstituteSurrogateAggregations extends OptimizerRules.OptimizerRule {
+ public SubstituteSurrogateAggregations() {
super(OptimizerRules.TransformDirection.UP);
}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteSurrogateExpressions.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteSurrogateExpressions.java
new file mode 100644
index 0000000000000..307634f4e8983
--- /dev/null
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/SubstituteSurrogateExpressions.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql.optimizer.rules.logical;
+
+import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.expression.SurrogateExpression;
+import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext;
+
+/**
+ * Replace {@link SurrogateExpression}s with their {@link SurrogateExpression#surrogate surrogates}.
+ */
+public final class SubstituteSurrogateExpressions extends OptimizerRules.OptimizerExpressionRule {
+
+ public SubstituteSurrogateExpressions() {
+ super(OptimizerRules.TransformDirection.UP);
+ }
+
+ @Override
+ protected Expression rule(Expression e, LogicalOptimizerContext ctx) {
+ return rule(e);
+ }
+
+ /**
+ * Perform the actual substitution.
+ */
+ public static Expression rule(Expression e) {
+ if (e instanceof SurrogateExpression s) {
+ Expression surrogate = s.surrogate();
+ if (surrogate != null) {
+ return surrogate;
+ }
+ }
+ return e;
+ }
+}
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java
index 11a8c5c5f5b98..6cde1f28535dd 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java
@@ -45,8 +45,8 @@
import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDouble;
import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToGeoPoint;
import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToGeoShape;
-import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToIP;
import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToInteger;
+import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToIpLeadingZerosRejected;
import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToLong;
import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToString;
import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToTimeDuration;
@@ -122,7 +122,7 @@ public class EsqlDataTypeConverter {
entry(GEO_POINT, ToGeoPoint::new),
entry(GEO_SHAPE, ToGeoShape::new),
entry(INTEGER, ToInteger::new),
- entry(IP, ToIP::new),
+ entry(IP, ToIpLeadingZerosRejected::new),
entry(LONG, ToLong::new),
// ToRadians, typeless
entry(KEYWORD, ToString::new),
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java
index 81022c5d69c35..e391b8ca1fa68 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java
@@ -13,10 +13,9 @@
import org.elasticsearch.xcontent.json.JsonXContent;
import org.elasticsearch.xpack.esql.LoadMapping;
import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
-import org.elasticsearch.xpack.esql.core.expression.Expression;
+import org.elasticsearch.xpack.esql.core.expression.function.Function;
import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
-import org.elasticsearch.xpack.esql.expression.function.FunctionDefinition;
import org.elasticsearch.xpack.esql.index.EsIndex;
import org.elasticsearch.xpack.esql.index.IndexResolution;
import org.elasticsearch.xpack.esql.parser.EsqlParser;
@@ -100,9 +99,9 @@ public void testInlineCast() throws IOException {
LogicalPlan plan = parser.createStatement("ROW a = 1::" + nameOrAlias);
Row row = as(plan, Row.class);
assertThat(row.fields(), hasSize(1));
- Expression functionCall = row.fields().get(0).child();
+ Function functionCall = (Function) row.fields().get(0).child();
assertThat(functionCall.dataType(), equalTo(expectedType));
- report.field(nameOrAlias, functionName(registry, functionCall));
+ report.field(nameOrAlias, registry.functionName(functionCall.getClass()));
}
report.endObject();
}
@@ -157,15 +156,6 @@ public void testJoinTwiceOnTheSameField_TwoLookups() {
);
}
- private String functionName(EsqlFunctionRegistry registry, Expression functionCall) {
- for (FunctionDefinition def : registry.listFunctions()) {
- if (functionCall.getClass().equals(def.clazz())) {
- return def.name();
- }
- }
- throw new IllegalArgumentException("can't find name for " + functionCall);
- }
-
private String error(String query) {
ParsingException e = expectThrows(ParsingException.class, () -> defaultAnalyzer.analyze(parser.createStatement(query)));
String message = e.getMessage();
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java
index d6f6b906a5d8f..e220eac5ab7a1 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java
@@ -42,6 +42,7 @@
import org.elasticsearch.xpack.esql.core.util.StringUtils;
import org.elasticsearch.xpack.esql.evaluator.EvalMapper;
import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper;
+import org.elasticsearch.xpack.esql.expression.SurrogateExpression;
import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Greatest;
import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce;
import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput;
@@ -484,13 +485,21 @@ protected final Expression buildDeepCopyOfFieldExpression(TestCaseSupplier.TestC
}
private Expression randomSerializeDeserialize(Expression expression) {
- if (randomBoolean()) {
+ if (canSerialize() == false || randomBoolean()) {
return expression;
}
return serializeDeserializeExpression(expression);
}
+ /**
+ * The expression being tested be serialized? The vast
+ * majority of expressions can be serialized.
+ */
+ protected boolean canSerialize() {
+ return true;
+ }
+
/**
* Returns the expression after being serialized and deserialized.
*
@@ -545,6 +554,12 @@ public static ExpressionEvaluator.Factory evaluator(Expression e) {
if (e.foldable()) {
e = new Literal(e.source(), e.fold(FoldContext.small()), e.dataType());
}
+ if (e instanceof SurrogateExpression s) {
+ Expression surrogate = s.surrogate();
+ if (surrogate != null) {
+ e = surrogate;
+ }
+ }
Layout.Builder builder = new Layout.Builder();
buildLayout(builder, e);
Expression.TypeResolution resolution = e.typeResolved();
@@ -705,6 +720,7 @@ private static BytesRef randomizeBytesRefOffset(BytesRef bytesRef) {
}
public void testSerializationOfSimple() {
+ assumeTrue("can't serialize function", canSerialize());
assertSerialization(buildFieldExpression(testCase), testCase.getConfiguration());
}
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java
index 0cee8fe3f57cb..f056e0c61c8d1 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java
@@ -25,6 +25,7 @@
import org.elasticsearch.xpack.esql.core.expression.FoldContext;
import org.elasticsearch.xpack.esql.core.type.DataType;
import org.elasticsearch.xpack.esql.core.util.NumericUtils;
+import org.elasticsearch.xpack.esql.expression.SurrogateExpression;
import org.elasticsearch.xpack.esql.optimizer.rules.logical.FoldNull;
import org.elasticsearch.xpack.esql.planner.PlannerUtils;
import org.hamcrest.Matcher;
@@ -366,6 +367,12 @@ public void testFold() {
return;
}
assertFalse("expected resolved", expression.typeResolved().unresolved());
+ if (expression instanceof SurrogateExpression s) {
+ Expression surrogate = s.surrogate();
+ if (surrogate != null) {
+ expression = surrogate;
+ }
+ }
Expression nullOptimized = new FoldNull().rule(expression, unboundLogicalOptimizerContext());
assertThat(nullOptimized.dataType(), equalTo(testCase.expectedType()));
assertTrue(nullOptimized.foldable());
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java
deleted file mode 100644
index e666d7c6defe9..0000000000000
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToIPTests.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
- */
-
-package org.elasticsearch.xpack.esql.expression.function.scalar.convert;
-
-import com.carrotsearch.randomizedtesting.annotations.Name;
-import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-
-import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.common.network.NetworkAddress;
-import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.tree.Source;
-import org.elasticsearch.xpack.esql.core.type.DataType;
-import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase;
-import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.function.Supplier;
-
-import static java.util.Collections.emptyList;
-import static org.elasticsearch.xpack.esql.core.util.StringUtils.parseIP;
-
-public class ToIPTests extends AbstractScalarFunctionTestCase {
- public ToIPTests(@Name("TestCase") Supplier testCaseSupplier) {
- this.testCase = testCaseSupplier.get();
- }
-
- @ParametersFactory
- public static Iterable