diff --git a/docs/reference/query-languages/esql/_snippets/functions/description/url_decode.md b/docs/reference/query-languages/esql/_snippets/functions/description/url_decode.md new file mode 100644 index 0000000000000..243c821782bd0 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/description/url_decode.md @@ -0,0 +1,6 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Description** + +URL decodes the input. + diff --git a/docs/reference/query-languages/esql/_snippets/functions/examples/url_decode.md b/docs/reference/query-languages/esql/_snippets/functions/examples/url_decode.md new file mode 100644 index 0000000000000..7dcc495a42732 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/examples/url_decode.md @@ -0,0 +1,13 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Example** + +```esql +ROW u = "https%3A%2F%2Fwww.example.com%2Fpapers%3Fq%3Dinformation%2Bretrieval%26year%3D2024%26citations%3Dhigh" | EVAL u = URL_DECODE(u) +``` + +| u:keyword | +| --- | +| https://www.example.com/papers?q=information+retrieval&year=2024&citations=high | + + diff --git a/docs/reference/query-languages/esql/_snippets/functions/layout/url_decode.md b/docs/reference/query-languages/esql/_snippets/functions/layout/url_decode.md new file mode 100644 index 0000000000000..c1d068dd621e0 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/layout/url_decode.md @@ -0,0 +1,27 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +## `URL_DECODE` [esql-url_decode] +```{applies_to} +stack: development +serverless: preview +``` + +**Syntax** + +:::{image} ../../../images/functions/url_decode.svg +:alt: Embedded +:class: text-center +::: + + +:::{include} ../parameters/url_decode.md +::: + +:::{include} ../description/url_decode.md +::: + +:::{include} ../types/url_decode.md +::: + +:::{include} ../examples/url_decode.md +::: diff --git a/docs/reference/query-languages/esql/_snippets/functions/parameters/url_decode.md b/docs/reference/query-languages/esql/_snippets/functions/parameters/url_decode.md new file mode 100644 index 0000000000000..e69055d67ec54 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/parameters/url_decode.md @@ -0,0 +1,7 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Parameters** + +`string` +: URL encoded string to decode. + diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/url_decode.md b/docs/reference/query-languages/esql/_snippets/functions/types/url_decode.md new file mode 100644 index 0000000000000..7221b9139e2b8 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/types/url_decode.md @@ -0,0 +1,9 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported types** + +| string | result | +| --- | --- | +| keyword | keyword | +| text | keyword | + diff --git a/docs/reference/query-languages/esql/images/functions/url_decode.svg b/docs/reference/query-languages/esql/images/functions/url_decode.svg new file mode 100644 index 0000000000000..cb4fd1465a3c3 --- /dev/null +++ b/docs/reference/query-languages/esql/images/functions/url_decode.svg @@ -0,0 +1 @@ +URL_DECODE(string) \ No newline at end of file diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/url_decode.json b/docs/reference/query-languages/esql/kibana/definition/functions/url_decode.json new file mode 100644 index 0000000000000..44dcef86adcb4 --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/definition/functions/url_decode.json @@ -0,0 +1,37 @@ +{ + "comment" : "This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it.", + "type" : "scalar", + "name" : "url_decode", + "description" : "URL decodes the input.", + "signatures" : [ + { + "params" : [ + { + "name" : "string", + "type" : "keyword", + "optional" : false, + "description" : "URL encoded string to decode." + } + ], + "variadic" : false, + "returnType" : "keyword" + }, + { + "params" : [ + { + "name" : "string", + "type" : "text", + "optional" : false, + "description" : "URL encoded string to decode." + } + ], + "variadic" : false, + "returnType" : "keyword" + } + ], + "examples" : [ + "ROW u = \"https%3A%2F%2Fwww.example.com%2Fpapers%3Fq%3Dinformation%2Bretrieval%26year%3D2024%26citations%3Dhigh\" | EVAL u = URL_DECODE(u)" + ], + "preview" : true, + "snapshot_only" : true +} diff --git a/docs/reference/query-languages/esql/kibana/docs/functions/url_decode.md b/docs/reference/query-languages/esql/kibana/docs/functions/url_decode.md new file mode 100644 index 0000000000000..41b423babbffd --- /dev/null +++ b/docs/reference/query-languages/esql/kibana/docs/functions/url_decode.md @@ -0,0 +1,8 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +### URL DECODE +URL decodes the input. + +```esql +ROW u = "https%3A%2F%2Fwww.example.com%2Fpapers%3Fq%3Dinformation%2Bretrieval%26year%3D2024%26citations%3Dhigh" | EVAL u = URL_DECODE(u) +``` diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec index d2fbc849318de..e681d09fb9aa6 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/string.csv-spec @@ -2587,3 +2587,73 @@ ROW u = ["hello elastic!", "a+b-c%d", "", "!#$&'()*+,/:;=?@[]"] | EVAL u = URL_E u:keyword ["hello+elastic%21", "a%2Bb-c%25d", "", "%21%23%24%26%27%28%29*%2B%2C%2F%3A%3B%3D%3F%40%5B%5D"] ; + +url_decode sample for docs +required_capability: url_decode + +// tag::url_decode[] +ROW u = "https%3A%2F%2Fwww.example.com%2Fpapers%3Fq%3Dinformation%2Bretrieval%26year%3D2024%26citations%3Dhigh" | EVAL u = URL_DECODE(u) +// end::url_decode[] +; + +// tag::url_decode-result[] +u:keyword +https://www.example.com/papers?q=information+retrieval&year=2024&citations=high +// end::url_decode-result[] +; + +url_decode mixed functions tests +required_capability: url_decode + +FROM employees +| WHERE emp_no == 10001 +| EVAL a = TRIM(URL_DECODE(first_name)) +| EVAL b = URL_DECODE(TO_LOWER(first_name)) +| KEEP a,b; + +a:keyword | b:keyword +Georgi | georgi +; + +url_decode mixed input tests +required_capability: url_decode + +ROW u = "%21%23%24%26%27%28%29*%2B%2C%2F%3A%3B%3D%3F%40%5B%5D" | EVAL u = URL_DECODE(u); + +u:keyword +"!#$&'()*+,/:;=?@[]" +; + +combined url encode decode tests with table reads +required_capability: url_encode +required_capability: url_decode + +FROM employees +| SORT emp_no +| LIMIT 10 +| EVAL name = URL_DECODE(URL_ENCODE(CONCAT(gender, " - ", first_name, "+", last_name, "; ", CONCAT("@", first_name, last_name)))) +| KEEP emp_no, name; + +emp_no:integer | name:keyword +10001 | M - Georgi+Facello; @GeorgiFacello +10002 | F - Bezalel+Simmel; @BezalelSimmel +10003 | M - Parto+Bamford; @PartoBamford +10004 | M - Chirstian+Koblick; @ChirstianKoblick +10005 | M - Kyoichi+Maliniak; @KyoichiMaliniak +10006 | F - Anneke+Preusig; @AnnekePreusig +10007 | F - Tzvetan+Zielinski; @TzvetanZielinski +10008 | M - Saniya+Kalloufi; @SaniyaKalloufi +10009 | F - Sumant+Peac; @SumantPeac +10010 | null +; + +combined url encode decode tests with random strings +required_capability: url_encode +required_capability: url_decode + +ROW u = ["https://www.example.com/papers?q=information+retrieval&year=2024&citations=high", "", "!#$&'()+/:;=?@[]", "💨🔥🪨💧"] +| eval u = URL_DECODE(URL_ENCODE(u)); + +u:keyword +["https://www.example.com/papers?q=information+retrieval&year=2024&citations=high", "", "!#$&'()+/:;=?@[]", "💨🔥🪨💧"] +; diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/UrlDecodeEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/UrlDecodeEvaluator.java new file mode 100644 index 0000000000000..10e073087dd3a --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/UrlDecodeEvaluator.java @@ -0,0 +1,153 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.OrdinalBytesRefVector; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link UrlDecode}. + * This class is generated. Edit {@code ConvertEvaluatorImplementer} instead. + */ +public final class UrlDecodeEvaluator extends AbstractConvertFunction.AbstractEvaluator { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(UrlDecodeEvaluator.class); + + private final EvalOperator.ExpressionEvaluator val; + + public UrlDecodeEvaluator(Source source, EvalOperator.ExpressionEvaluator val, + DriverContext driverContext) { + super(driverContext, source); + this.val = val; + } + + @Override + public EvalOperator.ExpressionEvaluator next() { + return val; + } + + @Override + public Block evalVector(Vector v) { + BytesRefVector vector = (BytesRefVector) v; + OrdinalBytesRefVector ordinals = vector.asOrdinals(); + if (ordinals != null) { + return evalOrdinals(ordinals); + } + int positionCount = v.getPositionCount(); + BytesRef scratchPad = new BytesRef(); + if (vector.isConstant()) { + return driverContext.blockFactory().newConstantBytesRefBlockWith(evalValue(vector, 0, scratchPad), positionCount); + } + try (BytesRefBlock.Builder builder = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + builder.appendBytesRef(evalValue(vector, p, scratchPad)); + } + return builder.build(); + } + } + + private BytesRef evalValue(BytesRefVector container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return UrlDecode.process(value); + } + + @Override + public Block evalBlock(Block b) { + BytesRefBlock block = (BytesRefBlock) b; + int positionCount = block.getPositionCount(); + try (BytesRefBlock.Builder builder = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { + BytesRef scratchPad = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + BytesRef value = evalValue(block, i, scratchPad); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendBytesRef(value); + valuesAppended = true; + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private BytesRef evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + BytesRef value = container.getBytesRef(index, scratchPad); + return UrlDecode.process(value); + } + + private Block evalOrdinals(OrdinalBytesRefVector v) { + int positionCount = v.getDictionaryVector().getPositionCount(); + BytesRef scratchPad = new BytesRef(); + try (BytesRefVector.Builder builder = driverContext.blockFactory().newBytesRefVectorBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + builder.appendBytesRef(evalValue(v.getDictionaryVector(), p, scratchPad)); + } + IntVector ordinals = v.getOrdinalsVector(); + ordinals.incRef(); + return new OrdinalBytesRefVector(ordinals, builder.build()).asBlock(); + } + } + + @Override + public String toString() { + return "UrlDecodeEvaluator[" + "val=" + val + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(val); + } + + @Override + public long baseRamBytesUsed() { + long baseRamBytesUsed = BASE_RAM_BYTES_USED; + baseRamBytesUsed += val.baseRamBytesUsed(); + return baseRamBytesUsed; + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory val; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory val) { + this.source = source; + this.val = val; + } + + @Override + public UrlDecodeEvaluator get(DriverContext context) { + return new UrlDecodeEvaluator(source, val.get(context), context); + } + + @Override + public String toString() { + return "UrlDecodeEvaluator[" + "val=" + val + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index f5b86af3c64f8..40649f0a5913b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1411,7 +1411,12 @@ public enum Cap { /** * URL encoding function. */ - URL_ENCODE(Build.current().isSnapshot()); + URL_ENCODE(Build.current().isSnapshot()), + + /** + * URL decoding function. + */ + URL_DECODE(Build.current().isSnapshot()); private final boolean enabled; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/ExpressionWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/ExpressionWritables.java index 25d131b23f623..1ee67d5f485f3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/ExpressionWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/ExpressionWritables.java @@ -34,6 +34,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToString; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToUnsignedLong; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToVersion; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.UrlDecode; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.UrlEncode; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Abs; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Acos; @@ -225,6 +226,7 @@ public static List unaryScalars() { entries.add(WildcardLikeList.ENTRY); entries.add(Delay.ENTRY); entries.add(UrlEncode.ENTRY); + entries.add(UrlDecode.ENTRY); // mv functions entries.addAll(MvFunctionWritables.getNamedWriteables()); return entries; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index bb98f4b2fbb20..6d8fa0ff4ace3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -83,6 +83,7 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToTimeDuration; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToUnsignedLong; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToVersion; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.UrlDecode; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.UrlEncode; import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateDiff; import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateExtract; @@ -517,7 +518,8 @@ private static FunctionDefinition[][] snapshotFunctions() { def(L2Norm.class, L2Norm::new, "v_l2_norm"), def(Magnitude.class, Magnitude::new, "v_magnitude"), def(Hamming.class, Hamming::new, "v_hamming"), - def(UrlEncode.class, UrlEncode::new, "url_encode") } }; + def(UrlEncode.class, UrlEncode::new, "url_encode"), + def(UrlDecode.class, UrlDecode::new, "url_decode") } }; } public EsqlFunctionRegistry snapshotRegistry() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/UrlDecode.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/UrlDecode.java new file mode 100644 index 0000000000000..1036811e9e839 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/UrlDecode.java @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.compute.ann.ConvertEvaluator; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesTo; +import org.elasticsearch.xpack.esql.expression.function.FunctionAppliesToLifecycle; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; + +import java.io.IOException; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; + +public final class UrlDecode extends UnaryScalarFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "UrlDecode", + UrlDecode::new + ); + + private UrlDecode(StreamInput in) throws IOException { + super(in); + } + + @FunctionInfo( + returnType = "keyword", + preview = true, + description = "URL decodes the input.", + examples = { @Example(file = "string", tag = "url_decode") }, + appliesTo = { @FunctionAppliesTo(lifeCycle = FunctionAppliesToLifecycle.DEVELOPMENT) } + ) + public UrlDecode( + Source source, + @Param(name = "string", type = { "keyword", "text" }, description = "URL encoded string to decode.") Expression str + ) { + super(source, str); + } + + @Override + public Expression replaceChildren(List newChildren) { + return new UrlDecode(source(), newChildren.get(0)); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, UrlDecode::new, field()); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + @Override + protected TypeResolution resolveType() { + if (childrenResolved() == false) { + return new TypeResolution("Unresolved children"); + } + return isString(field, sourceText(), TypeResolutions.ParamOrdinal.DEFAULT); + } + + @Override + public EvalOperator.ExpressionEvaluator.Factory toEvaluator(ToEvaluator toEvaluator) { + return new UrlDecodeEvaluator.Factory(source(), toEvaluator.apply(field())); + } + + @ConvertEvaluator() + static BytesRef process(final BytesRef val) { + String input = val.utf8ToString(); + String decoded = URLDecoder.decode(input, StandardCharsets.UTF_8); + return new BytesRef(decoded); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractUrlEncodeDecodeTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractUrlEncodeDecodeTestCase.java new file mode 100644 index 0000000000000..2c01868bb2cda --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractUrlEncodeDecodeTestCase.java @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.lucene.BytesRefs; +import org.elasticsearch.xpack.esql.core.type.DataType; + +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.function.Supplier; + +import static org.hamcrest.Matchers.equalTo; + +public abstract class AbstractUrlEncodeDecodeTestCase extends AbstractScalarFunctionTestCase { + + private record RandomUrl(String plain, String encoded) {} + + public static Iterable createParameters(boolean isEncoderTest) { + String evaluatorToString = isEncoderTest + ? "UrlEncodeEvaluator[val=Attribute[channel=0]]" + : "UrlDecodeEvaluator[val=Attribute[channel=0]]"; + + List suppliers = new ArrayList<>(); + + for (DataType dataType : DataType.stringTypes()) { + Supplier caseSupplier = () -> createTestCaseWithRandomUrl( + dataType, + evaluatorToString, + isEncoderTest + ); + + suppliers.add(new TestCaseSupplier(List.of(dataType), caseSupplier)); + + for (TestCaseSupplier.TypedDataSupplier supplier : TestCaseSupplier.stringCases(dataType)) { + TestCaseSupplier testCaseSupplier = new TestCaseSupplier( + supplier.name(), + List.of(supplier.type()), + () -> createTestCaseWithRandomString(dataType, evaluatorToString, isEncoderTest, supplier) + ); + suppliers.add(testCaseSupplier); + } + } + + return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(false, suppliers); + + } + + public static TestCaseSupplier.TestCase createTestCaseWithRandomUrl( + DataType dataType, + String evaluatorToString, + boolean isEncoderTest + ) { + RandomUrl url = generateRandomUrl(); + BytesRef input = new BytesRef(isEncoderTest ? url.plain() : url.encoded()); + BytesRef output = new BytesRef(isEncoderTest ? url.encoded() : url.plain()); + TestCaseSupplier.TypedData fieldTypedData = new TestCaseSupplier.TypedData(input, dataType, "string"); + + return new TestCaseSupplier.TestCase(List.of(fieldTypedData), evaluatorToString, dataType, equalTo(output)); + } + + public static TestCaseSupplier.TestCase createTestCaseWithRandomString( + DataType dataType, + String evaluatorToString, + boolean isEncoderTest, + TestCaseSupplier.TypedDataSupplier supplier + ) { + TestCaseSupplier.TypedData fieldTypedData = supplier.get(); + String plain = BytesRefs.toBytesRef(fieldTypedData.data()).utf8ToString(); + String encoded = encode(plain); + BytesRef input = new BytesRef(isEncoderTest ? plain : encoded); + BytesRef output = new BytesRef(isEncoderTest ? encoded : plain); + + return new TestCaseSupplier.TestCase( + List.of(new TestCaseSupplier.TypedData(input, dataType, "string")), + evaluatorToString, + dataType, + equalTo(output) + ); + } + + private static RandomUrl generateRandomUrl() { + String protocol = randomFrom("http://", "https://", ""); + String domain = String.format(Locale.ROOT, "%s.com", randomAlphaOfLengthBetween(3, 10)); + String path = randomFrom("", "/" + randomAlphanumericOfLength(5) + "/"); + String query = randomFrom("", "?" + randomAlphaOfLength(5) + "=" + randomAlphanumericOfLength(5)); + + String plain = String.format(Locale.ROOT, "%s%s%s%s", protocol, domain, path, query); + String encoded = encode(plain); + + return new RandomUrl(plain, encoded); + } + + private static String encode(String plain) { + return URLEncoder.encode(plain, StandardCharsets.UTF_8); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/UrlDecodeErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/UrlDecodeErrorTests.java new file mode 100644 index 0000000000000..3ba9d8623f8e4 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/UrlDecodeErrorTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.ErrorsForCasesWithoutExamplesTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; +import org.hamcrest.Matcher; + +import java.util.List; +import java.util.Set; + +import static org.hamcrest.Matchers.equalTo; + +public class UrlDecodeErrorTests extends ErrorsForCasesWithoutExamplesTestCase { + @Override + protected List cases() { + return paramsToSuppliers(UrlDecodeTests.parameters()); + } + + @Override + protected Expression build(Source source, List args) { + return new UrlDecode(source, args.get(0)); + } + + @Override + protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "string")); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/UrlDecodeSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/UrlDecodeSerializationTests.java new file mode 100644 index 0000000000000..39c0bbd5dd0a4 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/UrlDecodeSerializationTests.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractUnaryScalarSerializationTests; + +public class UrlDecodeSerializationTests extends AbstractUnaryScalarSerializationTests { + @Override + protected UrlDecode create(Source source, Expression child) { + return new UrlDecode(source, child); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/UrlDecodeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/UrlDecodeTests.java new file mode 100644 index 0000000000000..217ee206899d1 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/UrlDecodeTests.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.function.AbstractUrlEncodeDecodeTestCase; +import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; + +import java.util.List; +import java.util.function.Supplier; + +public class UrlDecodeTests extends AbstractUrlEncodeDecodeTestCase { + + public UrlDecodeTests(@Name("TestCase") Supplier testCaseSupplier) { + this.testCase = testCaseSupplier.get(); + } + + @ParametersFactory + public static Iterable parameters() { + return createParameters(false); + } + + @Override + protected Expression build(Source source, List args) { + return new UrlDecode(source, args.get(0)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/UrlEncodeTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/UrlEncodeTests.java index c09a44d4af572..ee57e71c30ba8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/UrlEncodeTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/UrlEncodeTests.java @@ -10,28 +10,15 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; -import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; -import org.elasticsearch.xpack.esql.expression.function.FunctionName; +import org.elasticsearch.xpack.esql.expression.function.AbstractUrlEncodeDecodeTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.List; -import java.util.Locale; import java.util.function.Supplier; -import static org.hamcrest.Matchers.equalTo; - -@FunctionName("url_encode") -public class UrlEncodeTests extends AbstractScalarFunctionTestCase { - - private record RandomUrl(String plain, String encoded) {} +public class UrlEncodeTests extends AbstractUrlEncodeDecodeTestCase { public UrlEncodeTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); @@ -39,68 +26,11 @@ public UrlEncodeTests(@Name("TestCase") Supplier test @ParametersFactory public static Iterable parameters() { - List suppliers = new ArrayList<>(); - - for (DataType dataType : DataType.stringTypes()) { - suppliers.add(new TestCaseSupplier(List.of(dataType), () -> createTestCaseWithRandomUrl(dataType))); - - for (TestCaseSupplier.TypedDataSupplier supplier : TestCaseSupplier.stringCases(dataType)) { - TestCaseSupplier testCaseSupplier = new TestCaseSupplier( - supplier.name(), - List.of(supplier.type()), - () -> createTestCaseWithRandomString(dataType, supplier) - ); - suppliers.add(testCaseSupplier); - } - } - - return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(false, suppliers); + return createParameters(true); } @Override protected Expression build(Source source, List args) { return new UrlEncode(source, args.get(0)); } - - private static TestCaseSupplier.TestCase createTestCaseWithRandomUrl(DataType dataType) { - RandomUrl url = generateRandomUrl(); - BytesRef input = new BytesRef(url.plain()); - BytesRef output = new BytesRef(url.encoded()); - TestCaseSupplier.TypedData fieldTypedData = new TestCaseSupplier.TypedData(input, dataType, "string"); - - return new TestCaseSupplier.TestCase( - List.of(fieldTypedData), - "UrlEncodeEvaluator[val=Attribute[channel=0]]", - dataType, - equalTo(output) - ); - } - - private static TestCaseSupplier.TestCase createTestCaseWithRandomString( - DataType dataType, - TestCaseSupplier.TypedDataSupplier supplier - ) { - TestCaseSupplier.TypedData fieldTypedData = supplier.get(); - BytesRef input = BytesRefs.toBytesRef(fieldTypedData.data()); - BytesRef output = new BytesRef(URLEncoder.encode(input.utf8ToString(), StandardCharsets.UTF_8)); - - return new TestCaseSupplier.TestCase( - List.of(fieldTypedData), - "UrlEncodeEvaluator[val=Attribute[channel=0]]", - dataType, - equalTo(output) - ); - } - - private static RandomUrl generateRandomUrl() { - String protocol = randomFrom("http://", "https://", ""); - String domain = String.format(Locale.ROOT, "%s.com", randomAlphaOfLengthBetween(3, 10)); - String path = randomFrom("", "/" + randomAlphanumericOfLength(5) + "/"); - String query = randomFrom("", "?" + randomAlphaOfLength(5) + "=" + randomAlphanumericOfLength(5)); - - String plain = String.format(Locale.ROOT, "%s%s%s%s", protocol, domain, path, query); - String encoded = URLEncoder.encode(plain, StandardCharsets.UTF_8); - - return new RandomUrl(plain, encoded); - } }