From 0ed23defed4e13a8485669279718de3f33f90ceb Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Wed, 15 Oct 2025 22:08:58 -0700 Subject: [PATCH 01/33] PPL tostring() implementation issue #4492 Signed-off-by: Asif Bashar --- .../sql/calcite/utils/PPLOperandTypes.java | 7 + .../function/PPLBuiltinOperators.java | 2 + .../expression/function/PPLFuncImpTable.java | 3 + .../function/udf/ToStringFunction.java | 162 +++++++++++++++ .../function/udf/ToStringFunctionTest.java | 149 ++++++++++++++ docs/user/ppl/functions/conversion.rst | 92 +++++++++ docs/user/ppl/functions/string.rst | 3 + ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 6 +- .../sql/ppl/parser/AstExpressionBuilder.java | 11 +- .../calcite/CalcitePPLStringFunctionTest.java | 185 ++++++++++++++++++ sql/src/main/antlr/OpenSearchSQLLexer.g4 | 1 + sql/src/main/antlr/OpenSearchSQLParser.g4 | 1 + 13 files changed, 621 insertions(+), 2 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java create mode 100644 core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java index 12a9a297542..69e1492538c 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java @@ -112,6 +112,13 @@ private PPLOperandTypes() {} SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER)); + public static final UDFOperandMetadata BOOLEAN_OR_NUMERIC_STRING_OR_STRING_STRING = + UDFOperandMetadata.wrap( + (CompositeOperandTypeChecker) + OperandTypes.family(SqlTypeFamily.BOOLEAN) + .or(OperandTypes.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.STRING)) + .or(OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING))); + public static final UDFOperandMetadata NUMERIC_NUMERIC_OPTIONAL_NUMERIC = UDFOperandMetadata.wrap( (CompositeOperandTypeChecker) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index 76f625ebd38..984f398b1ee 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -66,6 +66,7 @@ import org.opensearch.sql.expression.function.udf.RexExtractMultiFunction; import org.opensearch.sql.expression.function.udf.RexOffsetFunction; import org.opensearch.sql.expression.function.udf.SpanFunction; +import org.opensearch.sql.expression.function.udf.ToStringFunction; import org.opensearch.sql.expression.function.udf.condition.EarliestFunction; import org.opensearch.sql.expression.function.udf.condition.EnhancedCoalesceFunction; import org.opensearch.sql.expression.function.udf.condition.LatestFunction; @@ -411,6 +412,7 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { RELEVANCE_QUERY_FUNCTION_INSTANCE.toUDF("multi_match", false); public static final SqlOperator NUMBER_TO_STRING = new NumberToStringFunction().toUDF("NUMBER_TO_STRING"); + public static final SqlOperator TOSTRING = new ToStringFunction().toUDF("TOSTRING"); public static final SqlOperator WIDTH_BUCKET = new org.opensearch.sql.expression.function.udf.binning.WidthBucketFunction() .toUDF("WIDTH_BUCKET"); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index 8297ecf73ce..de89df383d2 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -210,6 +210,7 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIMESTAMPDIFF; import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIME_FORMAT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIME_TO_SEC; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TOSTRING; import static org.opensearch.sql.expression.function.BuiltinFunctionName.TO_DAYS; import static org.opensearch.sql.expression.function.BuiltinFunctionName.TO_SECONDS; import static org.opensearch.sql.expression.function.BuiltinFunctionName.TRANSFORM; @@ -887,6 +888,7 @@ void populate() { registerOperator(WEEKOFYEAR, PPLBuiltinOperators.WEEK); registerOperator(INTERNAL_PATTERN_PARSER, PPLBuiltinOperators.PATTERN_PARSER); + registerOperator(TOSTRING, PPLBuiltinOperators.TOSTRING); // Register MVJOIN to use Calcite's ARRAY_JOIN register( @@ -1058,6 +1060,7 @@ void populate() { SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER)), false)); + register( LOG, (FunctionImp2) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java new file mode 100644 index 00000000000..70d2a82b13c --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java @@ -0,0 +1,162 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.udf; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.text.NumberFormat; +import java.time.Duration; +import java.util.List; +import java.util.Locale; +import org.apache.calcite.adapter.enumerable.NotNullImplementor; +import org.apache.calcite.adapter.enumerable.NullPolicy; +import org.apache.calcite.adapter.enumerable.RexToLixTranslator; +import org.apache.calcite.linq4j.function.Strict; +import org.apache.calcite.linq4j.tree.Expression; +import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.opensearch.sql.calcite.utils.PPLOperandTypes; +import org.opensearch.sql.calcite.utils.PPLReturnTypes; +import org.opensearch.sql.expression.function.ImplementorUDF; +import org.opensearch.sql.expression.function.UDFOperandMetadata; + +/** + * A custom implementation of number/boolean to string . + * + *

This operator is necessary because tostring has following requirements "binary" Converts a + * number to a binary value. "hex" Converts the number to a hexadecimal value. "commas" Formats the + * number with commas. If the number includes a decimal, the function rounds the number to nearest + * two decimal places. "duration" Converts the value in seconds to the readable time format + * HH:MM:SS. if not format parameter provided, then consider value as boolean + */ +public class ToStringFunction extends ImplementorUDF { + public ToStringFunction() { + super( + new org.opensearch.sql.expression.function.udf.ToStringFunction.ToStringImplementor(), + NullPolicy.ANY); + } + + public static final String DURATION_FORMAT = "duration"; + public static final String HEX_FORMAT = "hex"; + public static final String COMMAS_FORMAT = "commas"; + public static final String BINARY_FORMAT = "binary"; + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return PPLReturnTypes.STRING_FORCE_NULLABLE; + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return PPLOperandTypes.BOOLEAN_OR_NUMERIC_STRING_OR_STRING_STRING; + } + + public static class ToStringImplementor implements NotNullImplementor { + + @Override + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + Expression fieldValue = translatedOperands.get(0); + if (translatedOperands.size() > 1) { + Expression format = translatedOperands.get(1); + return Expressions.call(ToStringFunction.class, "toString", fieldValue, format); + } else { + return Expressions.call(ToStringFunction.class, "toString", fieldValue); + } + } + } + + @Strict + public static String toString(boolean fieldValue) { + if (fieldValue) { + return "True"; + } else { + return "False"; + } + } + + @Strict + public static String toString(String fieldValue) { + return toString(Boolean.parseBoolean(fieldValue)); + } + + @Strict + public static String toString(BigDecimal num, String format) { + if (format.equals(DURATION_FORMAT)) { + Duration d = Duration.ofSeconds(num.toBigInteger().longValue()); + long hours = d.toHours(); + int minutes = d.toMinutesPart(); + int remainingSeconds = d.toSecondsPart(); + + String time_str = String.format("%02d:%02d:%02d", hours, minutes, remainingSeconds); + return time_str; + } else if (format.equals(HEX_FORMAT)) { + return num.toBigInteger().toString(16); + } else if (format.equals(COMMAS_FORMAT)) { + NumberFormat nf = NumberFormat.getNumberInstance(Locale.getDefault()); + nf.setMinimumFractionDigits(0); + nf.setMaximumFractionDigits(2); + return nf.format(num); + + } else if (format.equals(BINARY_FORMAT)) { + BigInteger integerPart = num.toBigInteger(); // 42 + return integerPart.toString(2); + } + return num.toString(); + } + + @Strict + public static String toString(double num, String format) { + if (format.equals(DURATION_FORMAT)) { + Duration d = Duration.ofSeconds(Math.round(num)); + long hours = d.toHours(); + int minutes = d.toMinutesPart(); + int remainingSeconds = d.toSecondsPart(); + String time_str = String.format("%02d:%02d:%02d", hours, minutes, remainingSeconds); + return time_str; + } else if (format.equals(HEX_FORMAT)) { + return Double.toHexString(num); + } else if (format.equals(COMMAS_FORMAT)) { + NumberFormat nf = NumberFormat.getNumberInstance(Locale.getDefault()); + return nf.format(num); + } else if (format.equals(BINARY_FORMAT)) { + return Long.toBinaryString(Double.doubleToLongBits(num)); + } + return Double.toString(num); + } + + @Strict + public static String toString(int num, String format) { + + if (format.equals(DURATION_FORMAT)) { + + int hours = num / 3600; + int minutes = (num % 3600) / 60; + int seconds = num % 60; + + String time_str = String.format("%02d:%02d:%02d", hours, minutes, seconds); + return time_str; + } else if (format.equals(HEX_FORMAT)) { + return Integer.toHexString(num); + } else if (format.equals(COMMAS_FORMAT)) { + NumberFormat nf = NumberFormat.getNumberInstance(Locale.getDefault()); + return nf.format(num); + } else if (format.equals(BINARY_FORMAT)) { + return Integer.toBinaryString(num); + } + return Integer.toString(num); + } + + @Strict + public static String toString(String str, String format) { + if (str.contains(".") || (str.length() > 10)) { + return toString(Double.parseDouble(str), format); + } else { + return toString(Integer.parseInt(str), format); + } + } +} diff --git a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java new file mode 100644 index 00000000000..cfd25796b23 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java @@ -0,0 +1,149 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.udf; + +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.util.Locale; + +import static org.junit.jupiter.api.Assertions.*; + +public class ToStringFunctionTest { + + private final ToStringFunction function = new ToStringFunction(); + + @Test + void testBooleanToString() { + assertEquals("True", ToStringFunction.toString(true)); + assertEquals("False", ToStringFunction.toString(false)); + } + + @Test + void testStringBooleanToString() { + assertEquals("True", ToStringFunction.toString("true")); + assertEquals("False", ToStringFunction.toString("false")); + assertEquals("False", ToStringFunction.toString("anythingElse")); + } + + @Test + void testBigDecimalToStringDurationFormat() { + BigDecimal num = new BigDecimal("3661"); // 1 hour 1 minute 1 second + String result = ToStringFunction.toString(num, ToStringFunction.DURATION_FORMAT); + assertEquals("01:01:01", result); + } + + @Test + void testBigDecimalToStringHexFormat() { + BigDecimal num = new BigDecimal("255"); + String result = ToStringFunction.toString(num, ToStringFunction.HEX_FORMAT); + assertEquals("ff", result); + } + + @Test + void testBigDecimalToStringCommasFormat() { + Locale.setDefault(Locale.US); // Ensure predictable comma placement + BigDecimal num = new BigDecimal("1234567.891"); + String result = ToStringFunction.toString(num, ToStringFunction.COMMAS_FORMAT); + assertTrue(result.contains(",")); + } + + @Test + void testBigDecimalToStringBinaryFormat() { + BigDecimal num = new BigDecimal("10"); + String result = ToStringFunction.toString(num, ToStringFunction.BINARY_FORMAT); + assertEquals("1010", result); + } + + @Test + void testBigDecimalToStringDefault() { + BigDecimal num = new BigDecimal("123.45"); + assertEquals("123.45", ToStringFunction.toString(num, "unknown")); + } + + @Test + void testDoubleToStringDurationFormat() { + double num = 3661.4; + String result = ToStringFunction.toString(num, ToStringFunction.DURATION_FORMAT); + assertEquals("01:01:01", result); + } + + @Test + void testDoubleToStringHexFormat() { + double num = 10.5; + String result = ToStringFunction.toString(num, ToStringFunction.HEX_FORMAT); + assertTrue(result.startsWith("0x")); + } + + @Test + void testDoubleToStringCommasFormat() { + Locale.setDefault(Locale.US); + double num = 12345.678; + String result = ToStringFunction.toString(num, ToStringFunction.COMMAS_FORMAT); + assertTrue(result.contains(",")); + } + + @Test + void testDoubleToStringBinaryFormat() { + double num = 10.0; + String result = ToStringFunction.toString(num, ToStringFunction.BINARY_FORMAT); + assertNotNull(result); + assertFalse(result.isEmpty()); + } + + @Test + void testDoubleToStringDefault() { + assertEquals("10.5", ToStringFunction.toString(10.5, "unknown")); + } + + @Test + void testIntToStringDurationFormat() { + int num = 3661; + String result = ToStringFunction.toString(num, ToStringFunction.DURATION_FORMAT); + assertEquals("01:01:01", result); + } + + @Test + void testIntToStringHexFormat() { + assertEquals("ff", ToStringFunction.toString(255, ToStringFunction.HEX_FORMAT)); + } + + @Test + void testIntToStringCommasFormat() { + Locale.setDefault(Locale.US); + String result = ToStringFunction.toString(1234567, ToStringFunction.COMMAS_FORMAT); + assertTrue(result.contains(",")); + } + + @Test + void testIntToStringBinaryFormat() { + assertEquals("1010", ToStringFunction.toString(10, ToStringFunction.BINARY_FORMAT)); + } + + @Test + void testIntToStringDefault() { + assertEquals("123", ToStringFunction.toString(123, "unknown")); + } + + @Test + void testStringNumericToStringIntFormat() { + String result = ToStringFunction.toString("42", ToStringFunction.HEX_FORMAT); + assertEquals("2a", result); + } + + @Test + void testStringNumericToStringDoubleFormat() { + String result = ToStringFunction.toString("42.5", ToStringFunction.COMMAS_FORMAT); + assertTrue(result.contains("42")); + } + + @Test + void testStringLargeNumberAsDouble() { + String largeNum = "1234567890123"; + String result = ToStringFunction.toString(largeNum, ToStringFunction.BINARY_FORMAT); + assertNotNull(result); + } +} diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index dbe4403540c..21124c27edc 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -78,3 +78,95 @@ Cast function can be chained:: |-------| | True | +-------+ + +TOSTRING +----- + +Description +>>>>>>>>>>> +There are two available usage based on paraemter types and number of parameters. +Usage with format type: tostring(number|string, string) converts the number in first argument to provided format type string in second argument. + Return type: string +Usage for boolean parameter without format type: tostring(boolean) converts the string to 'True' or 'False'. + Return type: string + +You can use this function with the eval commands and as part of eval expressions. +The first argument can be a number, number as string or boolean. +If first argument is a a number or number as string , second argument need to be format name. +If first argument is boolean, then second argument is not needed. + +format types: +a) "binary" Converts a number to a binary value. +b) "hex" Converts the number to a hexadecimal value. +c) "commas" Formats the number with commas. If the number includes a decimal, the function rounds the number to nearest two decimal places. +d) "duration" Converts the value in seconds to the readable time format HH:MM:SS. +The format argument is optional and is only used when the value argument is a number. The tostring function supports the following formats. + +Binary conversion +You can use this function to convert a number to a string of its binary representation. For example, the result of the following function is 1001, because the binary representation of 9 is 1001.: +eval result = tostring(9, "binary") + +For information about bitwise functions that you can use with the tostring function, see Bitwise functions. + +Basic examples +The following example returns "True 0xF 12,345.68". +... | eval n=tostring(1==1) + " " + tostring(15, "hex") + " " + tostring(12345.6789, "commas") +The following example returns foo=615 and foo2=00:10:15. The 615 seconds is converted into minutes and seconds. + +... | eval foo=615 | eval foo2 = tostring(foo, "duration") +The following example formats the column totalSales to display values with a currency symbol and commas. You must use a period between the currency value and the tostring function. + +Example:: + + os> source=people | eval `boolean_str` = tostring(1=1)| fields `boolean_str` + fetched rows / total rows = 1/1 + +---------------------+ + | boolean_str | + |---------------------+ + | True | + +---------------------+ + os> source=EMP | eval salary_binary = tostring(SAL, "binary") | fields ENAME, salary_binary, SAL" + fetched rows / total rows = 1/1 + +---------------+------------------+------------+ + | ENAME | salary_binary | SAL | + |---------------+------------------+------------+ + | SMITH | 1001110001000000 | 80000.00 | + +---------------+------------------+------------+ + os> source=EMP | eval salary_hex = tostring(SAL, "hex") | fields ENAME, salary_hex, SAL" + fetched rows / total rows = 1/1 + +---------------+------------------+------------+ + | ENAME | salary_hex | SAL | + |---------------+------------------+------------+ + | SMITH | 13880 | 80000.00 | + +---------------+---------------+---------------+ + + os> source=EMP | eval salary_commas = tostring(SAL, "commas") | fields ENAME, salary_commas, SAL" + fetched rows / total rows = 1/1 + +---------------+------------------+------------+ + | ENAME | salary_commas | SAL | + |---------------+------------------+------------+ + | SMITH | 80,000 | 80000.00 | + +---------------+------------------+------------+ + + + duration + + os> source=EMP | eval duration = tostring(6500, "duration") | fields ENAME, duration" + fetched rows / total rows = 1/1 + +---------------+-------------+ + | ENAME | duration | + |---------------+-------------+ + | SMITH | 01:48:20 | + +---------------+-------------+ + +Usage for boolean parameter without format type:: + +Example:: + + os> source=people | eval `boolean_str` = tostring(1=1)| fields `boolean_str` + fetched rows / total rows = 1/1 + +---------------------+ + | boolean_str | + |---------------------+ + | True | + +---------------------+ \ No newline at end of file diff --git a/docs/user/ppl/functions/string.rst b/docs/user/ppl/functions/string.rst index 24efa1434f5..43efed3c470 100644 --- a/docs/user/ppl/functions/string.rst +++ b/docs/user/ppl/functions/string.rst @@ -397,3 +397,6 @@ Example:: |---------------------+---------------------| | HELLOWORLD | HELLOWORLD | +---------------------+---------------------+ + + + diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index ba1e4960bb2..e528bb553ab 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -392,6 +392,7 @@ STRFTIME: 'STRFTIME'; // TEXT FUNCTIONS SUBSTR: 'SUBSTR'; SUBSTRING: 'SUBSTRING'; +TOSTRING: 'TOSTRING'; LTRIM: 'LTRIM'; RTRIM: 'RTRIM'; TRIM: 'TRIM'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index e13447b68e9..852c55c863c 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -847,11 +847,14 @@ evalFunctionCall : evalFunctionName LT_PRTHS functionArgs RT_PRTHS ; -// cast function + +// cast, tostring function dataTypeFunctionCall : CAST LT_PRTHS logicalExpression AS convertedDataType RT_PRTHS + | TOSTRING LT_PRTHS functionArgs RT_PRTHS ; + convertedDataType : typeName = DATE | typeName = TIME @@ -1434,6 +1437,7 @@ searchableKeyWord | USING | VALUE | CAST + | TOSTRING | GET_FORMAT | EXTRACT | INTERVAL diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index f037376f5c2..1460a3efc9a 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -414,7 +414,16 @@ private Function buildFunction( /** Cast function. */ @Override public UnresolvedExpression visitDataTypeFunctionCall(DataTypeFunctionCallContext ctx) { - return new Cast(visit(ctx.logicalExpression()), visit(ctx.convertedDataType())); + ParseTree rootNode = ctx.getChild(0); + String functionName = rootNode.getText(); + final String mappedName = + FUNCTION_NAME_MAPPING.getOrDefault(functionName.toLowerCase(Locale.ROOT), functionName); + + if (mappedName.equals("cast")) { + return new Cast(visit(ctx.logicalExpression()), visit(ctx.convertedDataType())); + } else { + return buildFunction(mappedName, ctx.functionArgs().functionArg()); + } } @Override diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java index 1e97052dea0..d41b2c22453 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java @@ -46,6 +46,191 @@ public void testLower() { verifyPPLToSparkSQL(root, expectedSparkSql); } + @Test + public void testToStringBoolean() { + String ppl = "source=EMP | eval boolean_value = tostring(1==1) | fields boolean_value |head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(boolean_value=[TOSTRING(true)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = "boolean_value=True\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `TOSTRING`(TRUE) `boolean_value`\n" + "FROM `scott`.`EMP`\n" + "LIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testToStringBin() { + String ppl = + "source=EMP | eval salary_binary = tostring(SAL, \"binary\") | fields ENAME," + + " salary_binary, SAL"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(ENAME=[$1], salary_binary=[TOSTRING($5, 'binary':VARCHAR)], SAL=[$5])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = + "ENAME=SMITH; salary_binary=1100100000; SAL=800.00\n" + + "ENAME=ALLEN; salary_binary=11001000000; SAL=1600.00\n" + + "ENAME=WARD; salary_binary=10011100010; SAL=1250.00\n" + + "ENAME=JONES; salary_binary=101110011111; SAL=2975.00\n" + + "ENAME=MARTIN; salary_binary=10011100010; SAL=1250.00\n" + + "ENAME=BLAKE; salary_binary=101100100010; SAL=2850.00\n" + + "ENAME=CLARK; salary_binary=100110010010; SAL=2450.00\n" + + "ENAME=SCOTT; salary_binary=101110111000; SAL=3000.00\n" + + "ENAME=KING; salary_binary=1001110001000; SAL=5000.00\n" + + "ENAME=TURNER; salary_binary=10111011100; SAL=1500.00\n" + + "ENAME=ADAMS; salary_binary=10001001100; SAL=1100.00\n" + + "ENAME=JAMES; salary_binary=1110110110; SAL=950.00\n" + + "ENAME=FORD; salary_binary=101110111000; SAL=3000.00\n" + + "ENAME=MILLER; salary_binary=10100010100; SAL=1300.00\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`(`SAL`, 'binary') `salary_binary`, `SAL`\nFROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testToStringHex() { + String ppl = + "source=EMP | eval salary_hex = tostring(SAL, \"hex\") | fields ENAME, salary_hex, SAL"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(ENAME=[$1], salary_hex=[TOSTRING($5, 'hex':VARCHAR)], SAL=[$5])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = + "ENAME=SMITH; salary_hex=320; SAL=800.00\n" + + "ENAME=ALLEN; salary_hex=640; SAL=1600.00\n" + + "ENAME=WARD; salary_hex=4e2; SAL=1250.00\n" + + "ENAME=JONES; salary_hex=b9f; SAL=2975.00\n" + + "ENAME=MARTIN; salary_hex=4e2; SAL=1250.00\n" + + "ENAME=BLAKE; salary_hex=b22; SAL=2850.00\n" + + "ENAME=CLARK; salary_hex=992; SAL=2450.00\n" + + "ENAME=SCOTT; salary_hex=bb8; SAL=3000.00\n" + + "ENAME=KING; salary_hex=1388; SAL=5000.00\n" + + "ENAME=TURNER; salary_hex=5dc; SAL=1500.00\n" + + "ENAME=ADAMS; salary_hex=44c; SAL=1100.00\n" + + "ENAME=JAMES; salary_hex=3b6; SAL=950.00\n" + + "ENAME=FORD; salary_hex=bb8; SAL=3000.00\n" + + "ENAME=MILLER; salary_hex=514; SAL=1300.00\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`(`SAL`, 'hex') `salary_hex`, `SAL`\nFROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testToStringHexFromNumberAsString() { + String ppl = + "source=EMP | eval salary_hex = tostring(\"1600\", \"hex\") | fields ENAME, salary_hex| head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n LogicalProject(ENAME=[$1], salary_hex=[TOSTRING('1600':VARCHAR, 'hex':VARCHAR)])\n LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = + "ENAME=SMITH; salary_hex=640\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`('1600', 'hex') `salary_hex`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testToStringCommaFromNumberAsString() { + String ppl = + "source=EMP | eval salary_comma = tostring(\"160040222\", \"commas\") | fields ENAME, salary_comma| head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n LogicalProject(ENAME=[$1], salary_comma=[TOSTRING('160040222':VARCHAR, 'commas':VARCHAR)])\n LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = + "ENAME=SMITH; salary_comma=160,040,222\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`('160040222', 'commas') `salary_comma`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + @Test + public void testToStringBinaryFromNumberAsString() { + String ppl = + "source=EMP | eval salary_binary = tostring(\"160040222\", \"binary\") | fields ENAME, salary_binary| head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n LogicalProject(ENAME=[$1], salary_binary=[TOSTRING('160040222':VARCHAR, 'binary':VARCHAR)])\n LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = + "ENAME=SMITH; salary_binary=1001100010100000010100011110\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`('160040222', 'binary') `salary_binary`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + @Test + public void testToStringCommas() { + String ppl = + "source=EMP | eval salary_commas = tostring(SAL, \"commas\") | fields ENAME," + + " salary_commas, SAL"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalProject(ENAME=[$1], salary_commas=[TOSTRING($5, 'commas':VARCHAR)], SAL=[$5])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = + "ENAME=SMITH; salary_commas=800; SAL=800.00\n" + + "ENAME=ALLEN; salary_commas=1,600; SAL=1600.00\n" + + "ENAME=WARD; salary_commas=1,250; SAL=1250.00\n" + + "ENAME=JONES; salary_commas=2,975; SAL=2975.00\n" + + "ENAME=MARTIN; salary_commas=1,250; SAL=1250.00\n" + + "ENAME=BLAKE; salary_commas=2,850; SAL=2850.00\n" + + "ENAME=CLARK; salary_commas=2,450; SAL=2450.00\n" + + "ENAME=SCOTT; salary_commas=3,000; SAL=3000.00\n" + + "ENAME=KING; salary_commas=5,000; SAL=5000.00\n" + + "ENAME=TURNER; salary_commas=1,500; SAL=1500.00\n" + + "ENAME=ADAMS; salary_commas=1,100; SAL=1100.00\n" + + "ENAME=JAMES; salary_commas=950; SAL=950.00\n" + + "ENAME=FORD; salary_commas=3,000; SAL=3000.00\n" + + "ENAME=MILLER; salary_commas=1,300; SAL=1300.00\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`(`SAL`, 'commas') `salary_commas`, `SAL`\nFROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testToStringDuration() { + String ppl = + "source=EMP | eval duration_commas = tostring(6500, \"duration\") | fields ENAME," + + " duration_commas|HEAD 1"; + + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(ENAME=[$1], duration_commas=[TOSTRING(6500, 'duration':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = "ENAME=SMITH; duration_commas=01:48:20\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`(6500, 'duration') `duration_commas`\n" + + "FROM `scott`.`EMP`\n" + + "LIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + @Test public void testLike() { String ppl = "source=EMP | where like(JOB, 'SALE%') | stats count() as cnt"; diff --git a/sql/src/main/antlr/OpenSearchSQLLexer.g4 b/sql/src/main/antlr/OpenSearchSQLLexer.g4 index ba7c5be85ab..6465c692da3 100644 --- a/sql/src/main/antlr/OpenSearchSQLLexer.g4 +++ b/sql/src/main/antlr/OpenSearchSQLLexer.g4 @@ -133,6 +133,7 @@ STDDEV_SAMP: 'STDDEV_SAMP'; SUBSTRING: 'SUBSTRING'; TRIM: 'TRIM'; +TOSTRING: 'TOSTRING'; // Keywords, but can be ID // Common Keywords, but can be ID diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index 5f7361160b3..fbaef12fb98 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -417,6 +417,7 @@ specificFunction : CASE expression caseFuncAlternative+ (ELSE elseArg = functionArg)? END # caseFunctionCall | CASE caseFuncAlternative+ (ELSE elseArg = functionArg)? END # caseFunctionCall | CAST '(' expression AS convertedDataType ')' # dataTypeFunctionCall + | TOSTRING '(' functionArg ')' # dataTypeFunctionCall ; relevanceFunction From 5a5b778cdf83230ff8a68a259a628e3ee0b3f318 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 16 Oct 2025 09:33:26 -0700 Subject: [PATCH 02/33] removed sql changes Signed-off-by: Asif Bashar --- sql/src/main/antlr/OpenSearchSQLLexer.g4 | 1 - sql/src/main/antlr/OpenSearchSQLParser.g4 | 1 - 2 files changed, 2 deletions(-) diff --git a/sql/src/main/antlr/OpenSearchSQLLexer.g4 b/sql/src/main/antlr/OpenSearchSQLLexer.g4 index 6465c692da3..ba7c5be85ab 100644 --- a/sql/src/main/antlr/OpenSearchSQLLexer.g4 +++ b/sql/src/main/antlr/OpenSearchSQLLexer.g4 @@ -133,7 +133,6 @@ STDDEV_SAMP: 'STDDEV_SAMP'; SUBSTRING: 'SUBSTRING'; TRIM: 'TRIM'; -TOSTRING: 'TOSTRING'; // Keywords, but can be ID // Common Keywords, but can be ID diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index fbaef12fb98..5f7361160b3 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -417,7 +417,6 @@ specificFunction : CASE expression caseFuncAlternative+ (ELSE elseArg = functionArg)? END # caseFunctionCall | CASE caseFuncAlternative+ (ELSE elseArg = functionArg)? END # caseFunctionCall | CAST '(' expression AS convertedDataType ')' # dataTypeFunctionCall - | TOSTRING '(' functionArg ')' # dataTypeFunctionCall ; relevanceFunction From 9d71a95f8487a7c5fa0e4cea8d11b4a3893efcaf Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 16 Oct 2025 09:50:53 -0700 Subject: [PATCH 03/33] doc changes Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 4 ++-- docs/user/ppl/functions/string.rst | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 21124c27edc..33af0075431 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -95,14 +95,14 @@ The first argument can be a number, number as string or boolean. If first argument is a a number or number as string , second argument need to be format name. If first argument is boolean, then second argument is not needed. -format types: +Format types: a) "binary" Converts a number to a binary value. b) "hex" Converts the number to a hexadecimal value. c) "commas" Formats the number with commas. If the number includes a decimal, the function rounds the number to nearest two decimal places. d) "duration" Converts the value in seconds to the readable time format HH:MM:SS. The format argument is optional and is only used when the value argument is a number. The tostring function supports the following formats. -Binary conversion +Binary conversion: You can use this function to convert a number to a string of its binary representation. For example, the result of the following function is 1001, because the binary representation of 9 is 1001.: eval result = tostring(9, "binary") diff --git a/docs/user/ppl/functions/string.rst b/docs/user/ppl/functions/string.rst index 43efed3c470..c1dd52a5d89 100644 --- a/docs/user/ppl/functions/string.rst +++ b/docs/user/ppl/functions/string.rst @@ -397,6 +397,4 @@ Example:: |---------------------+---------------------| | HELLOWORLD | HELLOWORLD | +---------------------+---------------------+ - - - + From be2c2e264a8cc337940f3f1ae2152a15672ce2cb Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 16 Oct 2025 09:52:09 -0700 Subject: [PATCH 04/33] docs changes Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 12 ++++++------ docs/user/ppl/functions/string.rst | 3 +-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 33af0075431..9d0bf6eab1b 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -85,15 +85,15 @@ TOSTRING Description >>>>>>>>>>> There are two available usage based on paraemter types and number of parameters. -Usage with format type: tostring(number|string, string) converts the number in first argument to provided format type string in second argument. - Return type: string +Usage with format type: tostring(ANY, [format]) converts the number in first argument to provided format type string in second argument. + Return type: string Usage for boolean parameter without format type: tostring(boolean) converts the string to 'True' or 'False'. - Return type: string + Return type: string You can use this function with the eval commands and as part of eval expressions. The first argument can be a number, number as string or boolean. -If first argument is a a number or number as string , second argument need to be format name. -If first argument is boolean, then second argument is not needed. +If first argument can be any valid type , second argument is optional and if provided , it needs to be format name. +If first argument is boolean, then second argument is not used even if its provided. Format types: a) "binary" Converts a number to a binary value. @@ -108,7 +108,7 @@ eval result = tostring(9, "binary") For information about bitwise functions that you can use with the tostring function, see Bitwise functions. -Basic examples +Basic examples: The following example returns "True 0xF 12,345.68". ... | eval n=tostring(1==1) + " " + tostring(15, "hex") + " " + tostring(12345.6789, "commas") The following example returns foo=615 and foo2=00:10:15. The 615 seconds is converted into minutes and seconds. diff --git a/docs/user/ppl/functions/string.rst b/docs/user/ppl/functions/string.rst index c1dd52a5d89..01b9b85b882 100644 --- a/docs/user/ppl/functions/string.rst +++ b/docs/user/ppl/functions/string.rst @@ -396,5 +396,4 @@ Example:: | UPPER('helloworld') | UPPER('HELLOWORLD') | |---------------------+---------------------| | HELLOWORLD | HELLOWORLD | - +---------------------+---------------------+ - + +---------------------+---------------------+ \ No newline at end of file From fc763a431cbbdcad3fe35ab3f0e70f334d0b12a6 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 16 Oct 2025 10:00:16 -0700 Subject: [PATCH 05/33] reverted string doc changes Signed-off-by: Asif Bashar --- docs/user/ppl/functions/string.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/ppl/functions/string.rst b/docs/user/ppl/functions/string.rst index 01b9b85b882..24efa1434f5 100644 --- a/docs/user/ppl/functions/string.rst +++ b/docs/user/ppl/functions/string.rst @@ -396,4 +396,4 @@ Example:: | UPPER('helloworld') | UPPER('HELLOWORLD') | |---------------------+---------------------| | HELLOWORLD | HELLOWORLD | - +---------------------+---------------------+ \ No newline at end of file + +---------------------+---------------------+ From a04eb14529029cf51d020e75ba5e7e6f921fb07c Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 16 Oct 2025 10:06:46 -0700 Subject: [PATCH 06/33] removed extra word Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 9d0bf6eab1b..58171d7cc72 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -91,7 +91,7 @@ Usage for boolean parameter without format type: tostring(boolean) converts the Return type: string You can use this function with the eval commands and as part of eval expressions. -The first argument can be a number, number as string or boolean. +The first argument can be a number, number as string or boolean. If first argument can be any valid type , second argument is optional and if provided , it needs to be format name. If first argument is boolean, then second argument is not used even if its provided. @@ -148,9 +148,6 @@ Example:: | SMITH | 80,000 | 80000.00 | +---------------+------------------+------------+ - - duration - os> source=EMP | eval duration = tostring(6500, "duration") | fields ENAME, duration" fetched rows / total rows = 1/1 +---------------+-------------+ From 590a8e63add13b75fa90047cd2eca1ab4d802936 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 16 Oct 2025 10:24:25 -0700 Subject: [PATCH 07/33] added any type Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 38 +++++++++++--------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 58171d7cc72..c4ebe0505de 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -85,27 +85,22 @@ TOSTRING Description >>>>>>>>>>> There are two available usage based on paraemter types and number of parameters. -Usage with format type: tostring(ANY, [format]) converts the number in first argument to provided format type string in second argument. +Usage with format type: tostring(ANY, [format]) converts the number in first argument to provided format type string in second argument. If non number type, then it converts to default string representation. Return type: string Usage for boolean parameter without format type: tostring(boolean) converts the string to 'True' or 'False'. Return type: string - You can use this function with the eval commands and as part of eval expressions. -The first argument can be a number, number as string or boolean. -If first argument can be any valid type , second argument is optional and if provided , it needs to be format name. + +If first argument can be any valid type , second argument is optional and if provided , it needs to be format name to convert to where first argument contains only numbers. If first argument is boolean, then second argument is not used even if its provided. -Format types: +Format types:: a) "binary" Converts a number to a binary value. b) "hex" Converts the number to a hexadecimal value. c) "commas" Formats the number with commas. If the number includes a decimal, the function rounds the number to nearest two decimal places. d) "duration" Converts the value in seconds to the readable time format HH:MM:SS. The format argument is optional and is only used when the value argument is a number. The tostring function supports the following formats. -Binary conversion: -You can use this function to convert a number to a string of its binary representation. For example, the result of the following function is 1001, because the binary representation of 9 is 1001.: -eval result = tostring(9, "binary") - For information about bitwise functions that you can use with the tostring function, see Bitwise functions. Basic examples: @@ -114,17 +109,11 @@ The following example returns "True 0xF 12,345.68". The following example returns foo=615 and foo2=00:10:15. The 615 seconds is converted into minutes and seconds. ... | eval foo=615 | eval foo2 = tostring(foo, "duration") -The following example formats the column totalSales to display values with a currency symbol and commas. You must use a period between the currency value and the tostring function. -Example:: - os> source=people | eval `boolean_str` = tostring(1=1)| fields `boolean_str` - fetched rows / total rows = 1/1 - +---------------------+ - | boolean_str | - |---------------------+ - | True | - +---------------------+ + +You can use this function to convert a number to a string of its binary representation. +Example:: os> source=EMP | eval salary_binary = tostring(SAL, "binary") | fields ENAME, salary_binary, SAL" fetched rows / total rows = 1/1 +---------------+------------------+------------+ @@ -132,6 +121,10 @@ Example:: |---------------+------------------+------------+ | SMITH | 1001110001000000 | 80000.00 | +---------------+------------------+------------+ + + +You can use this function to convert a number to a string of its hex representation. +Example:: os> source=EMP | eval salary_hex = tostring(SAL, "hex") | fields ENAME, salary_hex, SAL" fetched rows / total rows = 1/1 +---------------+------------------+------------+ @@ -140,6 +133,8 @@ Example:: | SMITH | 13880 | 80000.00 | +---------------+---------------+---------------+ +The following example formats the column totalSales to display values with commas. +Example:: os> source=EMP | eval salary_commas = tostring(SAL, "commas") | fields ENAME, salary_commas, SAL" fetched rows / total rows = 1/1 +---------------+------------------+------------+ @@ -147,7 +142,8 @@ Example:: |---------------+------------------+------------+ | SMITH | 80,000 | 80000.00 | +---------------+------------------+------------+ - +The following example converts number of seconds to HH:MM:SS format representing hours, minutes and seconds. +Example:: os> source=EMP | eval duration = tostring(6500, "duration") | fields ENAME, duration" fetched rows / total rows = 1/1 +---------------+-------------+ @@ -156,10 +152,8 @@ Example:: | SMITH | 01:48:20 | +---------------+-------------+ -Usage for boolean parameter without format type:: - +Example for boolean parameter. Example:: - os> source=people | eval `boolean_str` = tostring(1=1)| fields `boolean_str` fetched rows / total rows = 1/1 +---------------------+ From 6938e2c08dfef04119f1e37802d0a3a9554a0d21 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 16 Oct 2025 11:00:43 -0700 Subject: [PATCH 08/33] doc formatting fixes Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 38 ++++++++++++++++---------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index c4ebe0505de..280befee0aa 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -80,40 +80,44 @@ Cast function can be chained:: +-------+ TOSTRING ------ +----------- Description >>>>>>>>>>> -There are two available usage based on paraemter types and number of parameters. -Usage with format type: tostring(ANY, [format]) converts the number in first argument to provided format type string in second argument. If non number type, then it converts to default string representation. - Return type: string -Usage for boolean parameter without format type: tostring(boolean) converts the string to 'True' or 'False'. - Return type: string -You can use this function with the eval commands and as part of eval expressions. +The following usage options are available, depending on the parameter types and the number of parameters. -If first argument can be any valid type , second argument is optional and if provided , it needs to be format name to convert to where first argument contains only numbers. -If first argument is boolean, then second argument is not used even if its provided. +Usage with format type: tostring(ANY, [format]): Converts the number in first argument to provided format type string in second argument. If non number type, then it converts to default string representation. +Return type: string + +Usage for boolean parameter without format type tostring(boolean): Converts the string to 'True' or 'False'. +Return type: string + +You can use this function with the eval commands and as part of eval expressions. If first argument can be any valid type , second argument is optional and if provided , it needs to be format name to convert to where first argument contains only numbers. If first argument is boolean, then second argument is not used even if its provided. + +Format types: -Format types:: a) "binary" Converts a number to a binary value. b) "hex" Converts the number to a hexadecimal value. c) "commas" Formats the number with commas. If the number includes a decimal, the function rounds the number to nearest two decimal places. d) "duration" Converts the value in seconds to the readable time format HH:MM:SS. -The format argument is optional and is only used when the value argument is a number. The tostring function supports the following formats. -For information about bitwise functions that you can use with the tostring function, see Bitwise functions. +The format argument is optional and is only used when the value argument is a number. The tostring function supports the following formats. Basic examples: + The following example returns "True 0xF 12,345.68". -... | eval n=tostring(1==1) + " " + tostring(15, "hex") + " " + tostring(12345.6789, "commas") + + ... | eval n=tostring(1==1) + " " + tostring(15, "hex") + " " + tostring(12345.6789, "commas") + The following example returns foo=615 and foo2=00:10:15. The 615 seconds is converted into minutes and seconds. -... | eval foo=615 | eval foo2 = tostring(foo, "duration") + ... | eval foo=615 | eval foo2 = tostring(foo, "duration") You can use this function to convert a number to a string of its binary representation. Example:: + os> source=EMP | eval salary_binary = tostring(SAL, "binary") | fields ENAME, salary_binary, SAL" fetched rows / total rows = 1/1 +---------------+------------------+------------+ @@ -125,6 +129,7 @@ Example:: You can use this function to convert a number to a string of its hex representation. Example:: + os> source=EMP | eval salary_hex = tostring(SAL, "hex") | fields ENAME, salary_hex, SAL" fetched rows / total rows = 1/1 +---------------+------------------+------------+ @@ -135,6 +140,7 @@ Example:: The following example formats the column totalSales to display values with commas. Example:: + os> source=EMP | eval salary_commas = tostring(SAL, "commas") | fields ENAME, salary_commas, SAL" fetched rows / total rows = 1/1 +---------------+------------------+------------+ @@ -142,8 +148,10 @@ Example:: |---------------+------------------+------------+ | SMITH | 80,000 | 80000.00 | +---------------+------------------+------------+ + The following example converts number of seconds to HH:MM:SS format representing hours, minutes and seconds. Example:: + os> source=EMP | eval duration = tostring(6500, "duration") | fields ENAME, duration" fetched rows / total rows = 1/1 +---------------+-------------+ @@ -154,8 +162,10 @@ Example:: Example for boolean parameter. Example:: + os> source=people | eval `boolean_str` = tostring(1=1)| fields `boolean_str` fetched rows / total rows = 1/1 + +---------------------+ | boolean_str | |---------------------+ From 314fccdaee8ab28ede0eab2f1da4e2b83cc24e96 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 16 Oct 2025 11:04:06 -0700 Subject: [PATCH 09/33] description for boolean example Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 280befee0aa..036a89daf00 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -160,7 +160,7 @@ Example:: | SMITH | 01:48:20 | +---------------+-------------+ -Example for boolean parameter. +The following example for converts boolean parameter to string. Example:: os> source=people | eval `boolean_str` = tostring(1=1)| fields `boolean_str` From 0ee17b9a4a262a1d6070b98ae8a1162f6556f517 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Fri, 17 Oct 2025 13:21:49 -0700 Subject: [PATCH 10/33] added format_time call from calcite , added duration_millis as splunk default duration is in seconds which will be used for duration format , added cast call for tostring with 1 argument Signed-off-by: Asif Bashar --- .../sql/calcite/utils/PPLOperandTypes.java | 2 +- .../function/udf/ToStringFunction.java | 53 ++-- .../function/udf/ToStringFunctionTest.java | 265 +++++++++--------- .../sql/ppl/parser/AstExpressionBuilder.java | 52 +++- .../calcite/CalcitePPLStringFunctionTest.java | 157 +++++++---- 5 files changed, 312 insertions(+), 217 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java index 69e1492538c..20811f1af48 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java @@ -117,7 +117,7 @@ private PPLOperandTypes() {} (CompositeOperandTypeChecker) OperandTypes.family(SqlTypeFamily.BOOLEAN) .or(OperandTypes.family(SqlTypeFamily.NUMERIC, SqlTypeFamily.STRING)) - .or(OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING))); + .or(OperandTypes.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING))); public static final UDFOperandMetadata NUMERIC_NUMERIC_OPTIONAL_NUMERIC = UDFOperandMetadata.wrap( diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java index 70d2a82b13c..be76100f38d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToStringFunction.java @@ -8,7 +8,6 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.text.NumberFormat; -import java.time.Duration; import java.util.List; import java.util.Locale; import org.apache.calcite.adapter.enumerable.NotNullImplementor; @@ -18,6 +17,7 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.runtime.SqlFunctions; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PPLReturnTypes; @@ -41,9 +41,13 @@ public ToStringFunction() { } public static final String DURATION_FORMAT = "duration"; + public static final String DURATION_MILLIS_FORMAT = "duration_millis"; public static final String HEX_FORMAT = "hex"; public static final String COMMAS_FORMAT = "commas"; public static final String BINARY_FORMAT = "binary"; + public static final SqlFunctions.DateFormatFunction dateTimeFormatter = + new SqlFunctions.DateFormatFunction(); + public static final String format24hour = "%H:%M:%S"; // 24-hour format @Override public SqlReturnTypeInference getReturnTypeInference() { @@ -65,7 +69,13 @@ public Expression implement( Expression format = translatedOperands.get(1); return Expressions.call(ToStringFunction.class, "toString", fieldValue, format); } else { - return Expressions.call(ToStringFunction.class, "toString", fieldValue); + // autoboxes to Boolean + + if (!fieldValue.getType().getTypeName().equals("Boolean")) { + return Expressions.call(ToStringFunction.class, "toString", fieldValue); + } else { + return Expressions.call(ToStringFunction.class, "toString", fieldValue); + } } } } @@ -87,13 +97,13 @@ public static String toString(String fieldValue) { @Strict public static String toString(BigDecimal num, String format) { if (format.equals(DURATION_FORMAT)) { - Duration d = Duration.ofSeconds(num.toBigInteger().longValue()); - long hours = d.toHours(); - int minutes = d.toMinutesPart(); - int remainingSeconds = d.toSecondsPart(); - String time_str = String.format("%02d:%02d:%02d", hours, minutes, remainingSeconds); - return time_str; + return dateTimeFormatter.formatTime(format24hour, num.toBigInteger().intValue() * 1000); + + } else if (format.equals(DURATION_MILLIS_FORMAT)) { + + return dateTimeFormatter.formatTime(format24hour, num.toBigInteger().intValue()); + } else if (format.equals(HEX_FORMAT)) { return num.toBigInteger().toString(16); } else if (format.equals(COMMAS_FORMAT)) { @@ -112,12 +122,11 @@ public static String toString(BigDecimal num, String format) { @Strict public static String toString(double num, String format) { if (format.equals(DURATION_FORMAT)) { - Duration d = Duration.ofSeconds(Math.round(num)); - long hours = d.toHours(); - int minutes = d.toMinutesPart(); - int remainingSeconds = d.toSecondsPart(); - String time_str = String.format("%02d:%02d:%02d", hours, minutes, remainingSeconds); - return time_str; + return dateTimeFormatter.formatTime(format24hour, ((int) Math.round(num)) * 1000); + } else if (format.equals(DURATION_MILLIS_FORMAT)) { + + return dateTimeFormatter.formatTime(format24hour, ((int) Math.round(num))); + } else if (format.equals(HEX_FORMAT)) { return Double.toHexString(num); } else if (format.equals(COMMAS_FORMAT)) { @@ -129,17 +138,19 @@ public static String toString(double num, String format) { return Double.toString(num); } + @Strict + public static String toString(short num, String format) { + int i = (int) num; + return toString(i, format); + } + @Strict public static String toString(int num, String format) { if (format.equals(DURATION_FORMAT)) { - - int hours = num / 3600; - int minutes = (num % 3600) / 60; - int seconds = num % 60; - - String time_str = String.format("%02d:%02d:%02d", hours, minutes, seconds); - return time_str; + return dateTimeFormatter.formatTime(format24hour, num * 1000); + } else if (format.equals(DURATION_MILLIS_FORMAT)) { + return dateTimeFormatter.formatTime(format24hour, num); } else if (format.equals(HEX_FORMAT)) { return Integer.toHexString(num); } else if (format.equals(COMMAS_FORMAT)) { diff --git a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java index cfd25796b23..f0867f93327 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToStringFunctionTest.java @@ -5,145 +5,144 @@ package org.opensearch.sql.expression.function.udf; -import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; import java.math.BigDecimal; import java.util.Locale; - -import static org.junit.jupiter.api.Assertions.*; +import org.junit.jupiter.api.Test; public class ToStringFunctionTest { private final ToStringFunction function = new ToStringFunction(); - @Test - void testBooleanToString() { - assertEquals("True", ToStringFunction.toString(true)); - assertEquals("False", ToStringFunction.toString(false)); - } - - @Test - void testStringBooleanToString() { - assertEquals("True", ToStringFunction.toString("true")); - assertEquals("False", ToStringFunction.toString("false")); - assertEquals("False", ToStringFunction.toString("anythingElse")); - } - - @Test - void testBigDecimalToStringDurationFormat() { - BigDecimal num = new BigDecimal("3661"); // 1 hour 1 minute 1 second - String result = ToStringFunction.toString(num, ToStringFunction.DURATION_FORMAT); - assertEquals("01:01:01", result); - } - - @Test - void testBigDecimalToStringHexFormat() { - BigDecimal num = new BigDecimal("255"); - String result = ToStringFunction.toString(num, ToStringFunction.HEX_FORMAT); - assertEquals("ff", result); - } - - @Test - void testBigDecimalToStringCommasFormat() { - Locale.setDefault(Locale.US); // Ensure predictable comma placement - BigDecimal num = new BigDecimal("1234567.891"); - String result = ToStringFunction.toString(num, ToStringFunction.COMMAS_FORMAT); - assertTrue(result.contains(",")); - } - - @Test - void testBigDecimalToStringBinaryFormat() { - BigDecimal num = new BigDecimal("10"); - String result = ToStringFunction.toString(num, ToStringFunction.BINARY_FORMAT); - assertEquals("1010", result); - } - - @Test - void testBigDecimalToStringDefault() { - BigDecimal num = new BigDecimal("123.45"); - assertEquals("123.45", ToStringFunction.toString(num, "unknown")); - } - - @Test - void testDoubleToStringDurationFormat() { - double num = 3661.4; - String result = ToStringFunction.toString(num, ToStringFunction.DURATION_FORMAT); - assertEquals("01:01:01", result); - } - - @Test - void testDoubleToStringHexFormat() { - double num = 10.5; - String result = ToStringFunction.toString(num, ToStringFunction.HEX_FORMAT); - assertTrue(result.startsWith("0x")); - } - - @Test - void testDoubleToStringCommasFormat() { - Locale.setDefault(Locale.US); - double num = 12345.678; - String result = ToStringFunction.toString(num, ToStringFunction.COMMAS_FORMAT); - assertTrue(result.contains(",")); - } - - @Test - void testDoubleToStringBinaryFormat() { - double num = 10.0; - String result = ToStringFunction.toString(num, ToStringFunction.BINARY_FORMAT); - assertNotNull(result); - assertFalse(result.isEmpty()); - } - - @Test - void testDoubleToStringDefault() { - assertEquals("10.5", ToStringFunction.toString(10.5, "unknown")); - } - - @Test - void testIntToStringDurationFormat() { - int num = 3661; - String result = ToStringFunction.toString(num, ToStringFunction.DURATION_FORMAT); - assertEquals("01:01:01", result); - } - - @Test - void testIntToStringHexFormat() { - assertEquals("ff", ToStringFunction.toString(255, ToStringFunction.HEX_FORMAT)); - } - - @Test - void testIntToStringCommasFormat() { - Locale.setDefault(Locale.US); - String result = ToStringFunction.toString(1234567, ToStringFunction.COMMAS_FORMAT); - assertTrue(result.contains(",")); - } - - @Test - void testIntToStringBinaryFormat() { - assertEquals("1010", ToStringFunction.toString(10, ToStringFunction.BINARY_FORMAT)); - } - - @Test - void testIntToStringDefault() { - assertEquals("123", ToStringFunction.toString(123, "unknown")); - } - - @Test - void testStringNumericToStringIntFormat() { - String result = ToStringFunction.toString("42", ToStringFunction.HEX_FORMAT); - assertEquals("2a", result); - } - - @Test - void testStringNumericToStringDoubleFormat() { - String result = ToStringFunction.toString("42.5", ToStringFunction.COMMAS_FORMAT); - assertTrue(result.contains("42")); - } - - @Test - void testStringLargeNumberAsDouble() { - String largeNum = "1234567890123"; - String result = ToStringFunction.toString(largeNum, ToStringFunction.BINARY_FORMAT); - assertNotNull(result); - } + @Test + void testBooleanToString() { + assertEquals("True", ToStringFunction.toString(true)); + assertEquals("False", ToStringFunction.toString(false)); + } + + @Test + void testStringBooleanToString() { + assertEquals("True", ToStringFunction.toString("true")); + assertEquals("False", ToStringFunction.toString("false")); + assertEquals("False", ToStringFunction.toString("anythingElse")); + } + + @Test + void testBigDecimalToStringDurationFormat() { + BigDecimal num = new BigDecimal("3661"); // 1 hour 1 minute 1 second + String result = ToStringFunction.toString(num, ToStringFunction.DURATION_FORMAT); + assertEquals("01:01:01", result); + } + + @Test + void testBigDecimalToStringHexFormat() { + BigDecimal num = new BigDecimal("255"); + String result = ToStringFunction.toString(num, ToStringFunction.HEX_FORMAT); + assertEquals("ff", result); + } + + @Test + void testBigDecimalToStringCommasFormat() { + Locale.setDefault(Locale.US); // Ensure predictable comma placement + BigDecimal num = new BigDecimal("1234567.891"); + String result = ToStringFunction.toString(num, ToStringFunction.COMMAS_FORMAT); + assertTrue(result.contains(",")); + } + + @Test + void testBigDecimalToStringBinaryFormat() { + BigDecimal num = new BigDecimal("10"); + String result = ToStringFunction.toString(num, ToStringFunction.BINARY_FORMAT); + assertEquals("1010", result); + } + + @Test + void testBigDecimalToStringDefault() { + BigDecimal num = new BigDecimal("123.45"); + assertEquals("123.45", ToStringFunction.toString(num, "unknown")); + } + + @Test + void testDoubleToStringDurationFormat() { + double num = 3661.4; + String result = ToStringFunction.toString(num, ToStringFunction.DURATION_FORMAT); + assertEquals("01:01:01", result); + } + + @Test + void testDoubleToStringHexFormat() { + double num = 10.5; + String result = ToStringFunction.toString(num, ToStringFunction.HEX_FORMAT); + assertTrue(result.startsWith("0x")); + } + + @Test + void testDoubleToStringCommasFormat() { + Locale.setDefault(Locale.US); + double num = 12345.678; + String result = ToStringFunction.toString(num, ToStringFunction.COMMAS_FORMAT); + assertTrue(result.contains(",")); + } + + @Test + void testDoubleToStringBinaryFormat() { + double num = 10.0; + String result = ToStringFunction.toString(num, ToStringFunction.BINARY_FORMAT); + assertNotNull(result); + assertFalse(result.isEmpty()); + } + + @Test + void testDoubleToStringDefault() { + assertEquals("10.5", ToStringFunction.toString(10.5, "unknown")); + } + + @Test + void testIntToStringDurationFormat() { + int num = 3661; + String result = ToStringFunction.toString(num, ToStringFunction.DURATION_FORMAT); + assertEquals("01:01:01", result); + } + + @Test + void testIntToStringHexFormat() { + assertEquals("ff", ToStringFunction.toString(255, ToStringFunction.HEX_FORMAT)); + } + + @Test + void testIntToStringCommasFormat() { + Locale.setDefault(Locale.US); + String result = ToStringFunction.toString(1234567, ToStringFunction.COMMAS_FORMAT); + assertTrue(result.contains(",")); + } + + @Test + void testIntToStringBinaryFormat() { + assertEquals("1010", ToStringFunction.toString(10, ToStringFunction.BINARY_FORMAT)); + } + + @Test + void testIntToStringDefault() { + assertEquals("123", ToStringFunction.toString(123, "unknown")); + } + + @Test + void testStringNumericToStringIntFormat() { + String result = ToStringFunction.toString("42", ToStringFunction.HEX_FORMAT); + assertEquals("2a", result); + } + + @Test + void testStringNumericToStringDoubleFormat() { + String result = ToStringFunction.toString("42.5", ToStringFunction.COMMAS_FORMAT); + assertTrue(result.contains("42")); + } + + @Test + void testStringLargeNumberAsDouble() { + String largeNum = "1234567890123"; + String result = ToStringFunction.toString(largeNum, ToStringFunction.BINARY_FORMAT); + assertNotNull(result); + } } diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 1460a3efc9a..2112282f728 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -19,6 +19,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; +import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.RuleContext; import org.antlr.v4.runtime.tree.ParseTree; @@ -29,8 +30,10 @@ import org.opensearch.sql.ast.expression.subquery.ScalarSubquery; import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.calcite.plan.OpenSearchConstants; +import org.opensearch.sql.common.antlr.CaseInsensitiveCharStream; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.utils.StringUtils; +import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLLexer; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.BinaryArithmeticContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.BooleanLiteralContext; @@ -411,18 +414,51 @@ private Function buildFunction( functionName, args.stream().map(this::visitFunctionArg).collect(Collectors.toList())); } + public DataTypeFunctionCallContext createDataTypeFunctionCallContext(String castExpression) { + // Create a case-insensitive character stream from the input + CaseInsensitiveCharStream charStream = new CaseInsensitiveCharStream(castExpression); + + // Create lexer and parser + OpenSearchPPLLexer lexer = new OpenSearchPPLLexer(charStream); + CommonTokenStream tokens = new CommonTokenStream(lexer); + OpenSearchPPLParser parser = new OpenSearchPPLParser(tokens); + + // Parse the expression - cast is part of evalFunctionCall + DataTypeFunctionCallContext evalContext = parser.dataTypeFunctionCall(); + return evalContext; + } + /** Cast function. */ @Override public UnresolvedExpression visitDataTypeFunctionCall(DataTypeFunctionCallContext ctx) { - ParseTree rootNode = ctx.getChild(0); - String functionName = rootNode.getText(); - final String mappedName = - FUNCTION_NAME_MAPPING.getOrDefault(functionName.toLowerCase(Locale.ROOT), functionName); - - if (mappedName.equals("cast")) { - return new Cast(visit(ctx.logicalExpression()), visit(ctx.convertedDataType())); + if (ctx.functionArgs() != null) { + + ParseTree rootNode = ctx.getChild(0); + String functionName = rootNode.getText(); + final String mappedName = + FUNCTION_NAME_MAPPING.getOrDefault(functionName.toLowerCase(Locale.ROOT), functionName); + System.out.println(mappedName); + if (mappedName != null && mappedName.equals("tostring")) { + if (ctx.functionArgs().functionArg().size() == 1) { + List functionArgs = + ctx.functionArgs().functionArg(); + + String castExpresstion = + String.format("cast( %s as String)", functionArgs.getFirst().getText()); + DataTypeFunctionCallContext toStringDataTypeConversionContext = + this.createDataTypeFunctionCallContext(castExpresstion); + return new Cast( + visit(toStringDataTypeConversionContext.logicalExpression()), + visit(toStringDataTypeConversionContext.convertedDataType())); + // + } else { + return buildFunction(mappedName, ctx.functionArgs().functionArg()); + } + } else { + return new Cast(visit(ctx.logicalExpression()), visit(ctx.convertedDataType())); + } } else { - return buildFunction(mappedName, ctx.functionArgs().functionArg()); + return new Cast(visit(ctx.logicalExpression()), visit(ctx.convertedDataType())); } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java index d41b2c22453..b32ecd04e3f 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLStringFunctionTest.java @@ -46,20 +46,55 @@ public void testLower() { verifyPPLToSparkSQL(root, expectedSparkSql); } + // This test evalutes tostring where it gets converted to cast call + + @Test + public void testToStringFormatNotSpecified() { + String ppl = + "source=EMP | eval string_value = tostring(MGR) | eval cast_value = cast(MGR as string)|" + + " fields string_value, cast_value"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(string_value=[SAFE_CAST($3)], cast_value=[SAFE_CAST($3)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = + "string_value=7902; cast_value=7902\n" + + "string_value=7698; cast_value=7698\n" + + "string_value=7698; cast_value=7698\n" + + "string_value=7839; cast_value=7839\n" + + "string_value=7698; cast_value=7698\n" + + "string_value=7839; cast_value=7839\n" + + "string_value=7839; cast_value=7839\n" + + "string_value=7566; cast_value=7566\n" + + "string_value=null; cast_value=null\n" + + "string_value=7698; cast_value=7698\n" + + "string_value=7788; cast_value=7788\n" + + "string_value=7698; cast_value=7698\n" + + "string_value=7566; cast_value=7566\n" + + "string_value=7782; cast_value=7782\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT SAFE_CAST(`MGR` AS STRING) `string_value`, SAFE_CAST(`MGR` AS STRING)" + + " `cast_value`\n" + + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + @Test public void testToStringBoolean() { String ppl = "source=EMP | eval boolean_value = tostring(1==1) | fields boolean_value |head 1"; RelNode root = getRelNode(ppl); String expectedLogical = "LogicalSort(fetch=[1])\n" - + " LogicalProject(boolean_value=[TOSTRING(true)])\n" + + " LogicalProject(boolean_value=['TRUE':VARCHAR])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; - String expectedResult = "boolean_value=True\n"; + String expectedResult = "boolean_value=TRUE\n"; verifyLogical(root, expectedLogical); verifyResult(root, expectedResult); - String expectedSparkSql = - "SELECT `TOSTRING`(TRUE) `boolean_value`\n" + "FROM `scott`.`EMP`\n" + "LIMIT 1"; + String expectedSparkSql = "SELECT 'TRUE' `boolean_value`\nFROM `scott`.`EMP`\nLIMIT 1"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -126,56 +161,70 @@ public void testToStringHex() { verifyPPLToSparkSQL(root, expectedSparkSql); } - @Test - public void testToStringHexFromNumberAsString() { - String ppl = - "source=EMP | eval salary_hex = tostring(\"1600\", \"hex\") | fields ENAME, salary_hex| head 1"; - RelNode root = getRelNode(ppl); - String expectedLogical = - "LogicalSort(fetch=[1])\n LogicalProject(ENAME=[$1], salary_hex=[TOSTRING('1600':VARCHAR, 'hex':VARCHAR)])\n LogicalTableScan(table=[[scott, EMP]])\n"; - String expectedResult = - "ENAME=SMITH; salary_hex=640\n"; - verifyLogical(root, expectedLogical); - verifyResult(root, expectedResult); - - String expectedSparkSql = - "SELECT `ENAME`, `TOSTRING`('1600', 'hex') `salary_hex`\nFROM `scott`.`EMP`\nLIMIT 1"; - verifyPPLToSparkSQL(root, expectedSparkSql); - } - - @Test - public void testToStringCommaFromNumberAsString() { - String ppl = - "source=EMP | eval salary_comma = tostring(\"160040222\", \"commas\") | fields ENAME, salary_comma| head 1"; - RelNode root = getRelNode(ppl); - String expectedLogical = - "LogicalSort(fetch=[1])\n LogicalProject(ENAME=[$1], salary_comma=[TOSTRING('160040222':VARCHAR, 'commas':VARCHAR)])\n LogicalTableScan(table=[[scott, EMP]])\n"; - String expectedResult = - "ENAME=SMITH; salary_comma=160,040,222\n"; - verifyLogical(root, expectedLogical); - verifyResult(root, expectedResult); - - String expectedSparkSql = - "SELECT `ENAME`, `TOSTRING`('160040222', 'commas') `salary_comma`\nFROM `scott`.`EMP`\nLIMIT 1"; - verifyPPLToSparkSQL(root, expectedSparkSql); - } - @Test - public void testToStringBinaryFromNumberAsString() { - String ppl = - "source=EMP | eval salary_binary = tostring(\"160040222\", \"binary\") | fields ENAME, salary_binary| head 1"; - RelNode root = getRelNode(ppl); - String expectedLogical = - "LogicalSort(fetch=[1])\n LogicalProject(ENAME=[$1], salary_binary=[TOSTRING('160040222':VARCHAR, 'binary':VARCHAR)])\n LogicalTableScan(table=[[scott, EMP]])\n"; - String expectedResult = - "ENAME=SMITH; salary_binary=1001100010100000010100011110\n"; - verifyLogical(root, expectedLogical); - verifyResult(root, expectedResult); - - String expectedSparkSql = - "SELECT `ENAME`, `TOSTRING`('160040222', 'binary') `salary_binary`\nFROM `scott`.`EMP`\nLIMIT 1"; - verifyPPLToSparkSQL(root, expectedSparkSql); - } - @Test + @Test + public void testToStringHexFromNumberAsString() { + String ppl = + "source=EMP | eval salary_hex = tostring(\"1600\", \"hex\") | fields ENAME, salary_hex|" + + " head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(ENAME=[$1], salary_hex=[TOSTRING('1600':VARCHAR, 'hex':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = "ENAME=SMITH; salary_hex=640\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`('1600', 'hex') `salary_hex`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testToStringCommaFromNumberAsString() { + String ppl = + "source=EMP | eval salary_comma = tostring(\"160040222\", \"commas\") | fields ENAME," + + " salary_comma| head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(ENAME=[$1], salary_comma=[TOSTRING('160040222':VARCHAR," + + " 'commas':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = "ENAME=SMITH; salary_comma=160,040,222\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`('160040222', 'commas') `salary_comma`\n" + + "FROM `scott`.`EMP`\n" + + "LIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testToStringBinaryFromNumberAsString() { + String ppl = + "source=EMP | eval salary_binary = tostring(\"160040222\", \"binary\") | fields ENAME," + + " salary_binary| head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(ENAME=[$1], salary_binary=[TOSTRING('160040222':VARCHAR," + + " 'binary':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + String expectedResult = "ENAME=SMITH; salary_binary=1001100010100000010100011110\n"; + verifyLogical(root, expectedLogical); + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `ENAME`, `TOSTRING`('160040222', 'binary') `salary_binary`\n" + + "FROM `scott`.`EMP`\n" + + "LIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test public void testToStringCommas() { String ppl = "source=EMP | eval salary_commas = tostring(SAL, \"commas\") | fields ENAME," From 6e24aa3c4b16f6a5734db18f210b8b5c1f5f245f Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Fri, 17 Oct 2025 13:32:18 -0700 Subject: [PATCH 11/33] added doc update to specifically set 2nd argument as optional Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 036a89daf00..9f9646efef4 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -86,10 +86,10 @@ Description >>>>>>>>>>> The following usage options are available, depending on the parameter types and the number of parameters. -Usage with format type: tostring(ANY, [format]): Converts the number in first argument to provided format type string in second argument. If non number type, then it converts to default string representation. +Usage with format type: tostring(ANY, [format]): Converts the number in first argument to provided format type string in second argument. If second argument is not provided, then it converts to default string representation. Return type: string -Usage for boolean parameter without format type tostring(boolean): Converts the string to 'True' or 'False'. +Usage for boolean parameter without format type tostring(boolean): Converts the string to 'TRUE' or 'FALSE'. Return type: string You can use this function with the eval commands and as part of eval expressions. If first argument can be any valid type , second argument is optional and if provided , it needs to be format name to convert to where first argument contains only numbers. If first argument is boolean, then second argument is not used even if its provided. @@ -100,12 +100,13 @@ a) "binary" Converts a number to a binary value. b) "hex" Converts the number to a hexadecimal value. c) "commas" Formats the number with commas. If the number includes a decimal, the function rounds the number to nearest two decimal places. d) "duration" Converts the value in seconds to the readable time format HH:MM:SS. +5) "duration_millis" Converts the value in milliseconds to the readable time format HH:MM:SS. The format argument is optional and is only used when the value argument is a number. The tostring function supports the following formats. Basic examples: -The following example returns "True 0xF 12,345.68". +The following example returns "TRUE 0xF 12,345.68". ... | eval n=tostring(1==1) + " " + tostring(15, "hex") + " " + tostring(12345.6789, "commas") @@ -169,5 +170,5 @@ Example:: +---------------------+ | boolean_str | |---------------------+ - | True | + | TRUE | +---------------------+ \ No newline at end of file From 454cfc8db324866376f2452290b0fd0739008754 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Fri, 17 Oct 2025 13:33:59 -0700 Subject: [PATCH 12/33] mentioned as value instead of number specifically Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 9f9646efef4..4f2d432c71a 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -86,7 +86,7 @@ Description >>>>>>>>>>> The following usage options are available, depending on the parameter types and the number of parameters. -Usage with format type: tostring(ANY, [format]): Converts the number in first argument to provided format type string in second argument. If second argument is not provided, then it converts to default string representation. +Usage with format type: tostring(ANY, [format]): Converts the value in first argument to provided format type string in second argument. If second argument is not provided, then it converts to default string representation. Return type: string Usage for boolean parameter without format type tostring(boolean): Converts the string to 'TRUE' or 'FALSE'. From b221e8c4f583b947198253ee90b5edb6f15b382a Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Fri, 17 Oct 2025 13:46:56 -0700 Subject: [PATCH 13/33] fixed wrong bullet point Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 4f2d432c71a..708bde1fa11 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -100,7 +100,7 @@ a) "binary" Converts a number to a binary value. b) "hex" Converts the number to a hexadecimal value. c) "commas" Formats the number with commas. If the number includes a decimal, the function rounds the number to nearest two decimal places. d) "duration" Converts the value in seconds to the readable time format HH:MM:SS. -5) "duration_millis" Converts the value in milliseconds to the readable time format HH:MM:SS. +e) "duration_millis" Converts the value in milliseconds to the readable time format HH:MM:SS. The format argument is optional and is only used when the value argument is a number. The tostring function supports the following formats. From ea9ba0fa8b5b63053df344930dc218e83dc75f91 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Mon, 20 Oct 2025 12:41:39 -0700 Subject: [PATCH 14/33] added more unit tests Signed-off-by: Asif Bashar --- .../function/udf/ToNumberFunctionTest.java | 192 ++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java diff --git a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java new file mode 100644 index 00000000000..d7f2b6fd685 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java @@ -0,0 +1,192 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression.function.udf; + +import static org.junit.jupiter.api.Assertions.*; + +import java.math.BigDecimal; +import java.util.Locale; +import org.apache.calcite.sql.type.ReturnTypes; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.calcite.utils.PPLOperandTypes; + +public class ToNumberFunctionTest { + + private final ToNumberFunction function = new ToNumberFunction(); + + @Test + void testGetReturnTypeInference() { + assertEquals(ReturnTypes.DOUBLE_FORCE_NULLABLE, function.getReturnTypeInference()); + } + + @Test + void testGetOperandMetadata() { + assertEquals(PPLOperandTypes.STRING_OR_STRING_INTEGER, function.getOperandMetadata()); + } + + @Test + void testToNumberWithDefaultBase() { + assertEquals(123, ToNumberFunction.toNumber("123")); + assertEquals(0, ToNumberFunction.toNumber("0")); + assertEquals(-456, ToNumberFunction.toNumber("-456")); + assertEquals(123.45, ToNumberFunction.toNumber("123.45")); + assertEquals(-123.45, ToNumberFunction.toNumber("-123.45")); + assertEquals(0.5, ToNumberFunction.toNumber("0.5")); + assertEquals(-0.5, ToNumberFunction.toNumber("-0.5")); + } + + @Test + void testToNumberWithBase10() { + assertEquals(123, ToNumberFunction.toNumber("123", 10)); + assertEquals(0, ToNumberFunction.toNumber("0", 10)); + assertEquals(-456, ToNumberFunction.toNumber("-456", 10)); + assertEquals(123.45, ToNumberFunction.toNumber("123.45", 10)); + assertEquals(-123.45, ToNumberFunction.toNumber("-123.45", 10)); + } + + @Test + void testToNumberWithBase2() { + assertEquals(5, ToNumberFunction.toNumber("101", 2)); + assertEquals(0, ToNumberFunction.toNumber("0", 2)); + assertEquals(1, ToNumberFunction.toNumber("1", 2)); + assertEquals(7, ToNumberFunction.toNumber("111", 2)); + assertEquals(10, ToNumberFunction.toNumber("1010", 2)); + } + + @Test + void testToNumberWithBase8() { + assertEquals(64, ToNumberFunction.toNumber("100", 8)); + assertEquals(8, ToNumberFunction.toNumber("10", 8)); + assertEquals(83, ToNumberFunction.toNumber("123", 8)); + assertEquals(511, ToNumberFunction.toNumber("777", 8)); + } + + @Test + void testToNumberWithBase16() { + assertEquals(255, ToNumberFunction.toNumber("FF", 16)); + assertEquals(16, ToNumberFunction.toNumber("10", 16)); + assertEquals(171, ToNumberFunction.toNumber("AB", 16)); + assertEquals(291, ToNumberFunction.toNumber("123", 16)); + assertEquals(4095, ToNumberFunction.toNumber("FFF", 16)); + } + + @Test + void testToNumberWithBase36() { + assertEquals(35, ToNumberFunction.toNumber("Z", 36)); + assertEquals(1295, ToNumberFunction.toNumber("ZZ", 36)); + assertEquals(46655, ToNumberFunction.toNumber("ZZZ", 36)); + } + + @Test + void testToNumberWithDecimalBase2() { + assertEquals(2.5, ToNumberFunction.toNumber("10.1", 2)); + assertEquals(1.5, ToNumberFunction.toNumber("1.1", 2)); + assertEquals(3.75, ToNumberFunction.toNumber("11.11", 2)); + } + + @Test + void testToNumberWithDecimalBase16() { + assertEquals(255.5, ToNumberFunction.toNumber("FF.8", 16)); + assertEquals(16.25, ToNumberFunction.toNumber("10.4", 16)); + assertEquals(171.6875, ToNumberFunction.toNumber("AB.B", 16)); + } + + @Test + void testToNumberWithNegativeDecimal() { + assertEquals(-2.5, ToNumberFunction.toNumber("-10.1", 2)); + assertEquals(-255.5, ToNumberFunction.toNumber("-FF.8", 16)); + assertEquals(-123.45, ToNumberFunction.toNumber("-123.45", 10)); + } + + @Test + void testToNumberWithEmptyFractionalPart() { + assertEquals(123.0, ToNumberFunction.toNumber("123.", 10)); + assertEquals(255.0, ToNumberFunction.toNumber("FF.", 16)); + assertEquals(5.0, ToNumberFunction.toNumber("101.", 2)); + } + + @Test + void testToNumberWithZeroIntegerPart() { + assertEquals(0.5, ToNumberFunction.toNumber("0.5", 10)); + assertEquals(0.5, ToNumberFunction.toNumber("0.1", 2)); + } + + @Test + void testToNumberInvalidBase() { + assertThrows(IllegalArgumentException.class, () -> { + ToNumberFunction.toNumber("123", 1); + }); + + assertThrows(IllegalArgumentException.class, () -> { + ToNumberFunction.toNumber("123", 37); + }); + + assertThrows(IllegalArgumentException.class, () -> { + ToNumberFunction.toNumber("123", 0); + }); + + assertThrows(IllegalArgumentException.class, () -> { + ToNumberFunction.toNumber("123", -1); + }); + } + + @Test + void testToNumberInvalidDigits() { + assertThrows(IllegalArgumentException.class, () -> { + ToNumberFunction.toNumber("12A", 10); + }); + + assertThrows(IllegalArgumentException.class, () -> { + ToNumberFunction.toNumber("102", 2); + }); + + assertThrows(IllegalArgumentException.class, () -> { + ToNumberFunction.toNumber("189", 8); + }); + + assertThrows(IllegalArgumentException.class, () -> { + ToNumberFunction.toNumber("GHI", 16); + }); + } + + @Test + void testToNumberInvalidFractionalDigits() { + assertThrows(IllegalArgumentException.class, () -> { + ToNumberFunction.toNumber("10.2", 2); + }); + + assertThrows(IllegalArgumentException.class, () -> { + ToNumberFunction.toNumber("FF.G", 16); + }); + + assertThrows(IllegalArgumentException.class, () -> { + ToNumberFunction.toNumber("123.ABC", 10); + }); + } + + @Test + void testToNumberEdgeCases() { + assertEquals(0, ToNumberFunction.toNumber("0", 2)); + assertEquals(0, ToNumberFunction.toNumber("0", 36)); + assertEquals(0.0, ToNumberFunction.toNumber("0.0", 10)); + assertEquals(0.0, ToNumberFunction.toNumber("0.000", 10)); + } + + @Test + void testToNumberLargeNumbers() { + assertEquals(Integer.MAX_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MAX_VALUE), 10)); + assertEquals(Integer.MIN_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MIN_VALUE), 10)); + } + + @Test + void testToNumberCaseInsensitivity() { + assertEquals(255, ToNumberFunction.toNumber("ff", 16)); + assertEquals(255, ToNumberFunction.toNumber("FF", 16)); + assertEquals(255, ToNumberFunction.toNumber("fF", 16)); + assertEquals(171, ToNumberFunction.toNumber("ab", 16)); + assertEquals(171, ToNumberFunction.toNumber("AB", 16)); + } +} From 370b9bc5d30ec90fbd7dc668ca5ee0939a720f87 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Oct 2025 21:03:42 -0700 Subject: [PATCH 15/33] Update docs/user/ppl/functions/conversion.rst Co-authored-by: ritvibhatt <53196324+ritvibhatt@users.noreply.github.com> Signed-off-by: Asif Bashar Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index fad37c0bf26..d9da1bf3d8e 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -86,7 +86,7 @@ Description >>>>>>>>>>> The following usage options are available, depending on the parameter types and the number of parameters. -Usage with format type: tonumber(string, [base]): Converts the value in first argument to provided base type string in second argument. If second argument is not provided, then it converts to base 10 number representation. +Usage: tonumber(string, [base]) converts the value in first argument to provided base type string in second argument. If second argument is not provided, then it converts to base 10 number representation. Return type: Number From f0706849c4446e47615a2228debbea69c069d0b0 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Oct 2025 21:20:00 -0700 Subject: [PATCH 16/33] fix per recommendation Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 34 ++++++++----------- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 4 +-- .../sql/ppl/parser/AstExpressionBuilder.java | 32 ----------------- 3 files changed, 16 insertions(+), 54 deletions(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index d9da1bf3d8e..3264d422ce1 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -84,39 +84,34 @@ TONUMBER Description >>>>>>>>>>> + The following usage options are available, depending on the parameter types and the number of parameters. Usage: tonumber(string, [base]) converts the value in first argument to provided base type string in second argument. If second argument is not provided, then it converts to base 10 number representation. + Return type: Number You can use this function with the eval commands and as part of eval expressions. Base values can be between 2 and 36. - - -Basic examples: You can use this function to convert a string representation of a binary number to return the corresponding number in base 10. -For example, the result of the following function is 5: -eval result = tonumber("0101", 2) - +Following example converts a string in binary to the number representation:: -Example:: -Following example converts a string in binary to the number representation. - os> source=EMP | eval int_value = tonumber('010101',2) | fields int_value|head 1 + os> source=EMP | eval int_value = tonumber('010101',2) | fields int_value | head 1 fetched rows / total rows = 1/1 - +--------------+ - | int_value | - |--------------+ - | 21.0 | - +--------------+ + +---------------+ + | int_value | + |---------------+ + | 21.0 | + +---------------+ + +Following example converts a string in hex to the number representation:: -Following example converts a string in hex to the number representation. -Example:: - os> source=EMP | eval int_value = tonumber('FA34',16) | fields int_value|head 1 + os> source=EMP | eval int_value = tonumber('FA34',16) | fields int_value | head 1 fetched rows / total rows = 1/1 +---------------+ | int_value | @@ -124,10 +119,9 @@ Example:: | 64052.0 | +---------------+ -Following example converts a string in decimal to the number representation. -Example:: +Following example converts a string in decimal to the number representation:: - os> source=EMP | eval int_value = tonumber('4598') | fields int_value|head 1 + os> source=EMP | eval int_value = tonumber('4598') | fields int_value | head 1 fetched rows / total rows = 1/1 +---------------+ | int_value | diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index e974c303375..a26b7381797 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -848,10 +848,9 @@ evalFunctionCall ; -// cast, tostring function +// cast function dataTypeFunctionCall : CAST LT_PRTHS logicalExpression AS convertedDataType RT_PRTHS - | TONUMBER LT_PRTHS functionArgs RT_PRTHS ; @@ -1234,6 +1233,7 @@ textFunctionName | LOCATE | REPLACE | REVERSE + | TONUMBER ; positionFunctionName diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 244f287c77a..b70226d039a 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -431,39 +431,7 @@ public DataTypeFunctionCallContext createDataTypeFunctionCallContext(String cast /** Cast function. */ @Override public UnresolvedExpression visitDataTypeFunctionCall(DataTypeFunctionCallContext ctx) { - if (ctx.functionArgs() != null) { - - ParseTree rootNode = ctx.getChild(0); - String functionName = rootNode.getText(); - final String mappedName = - FUNCTION_NAME_MAPPING.getOrDefault(functionName.toLowerCase(Locale.ROOT), functionName); - System.out.println(mappedName); - if (mappedName != null && mappedName.equals("tostring")) { - if (ctx.functionArgs().functionArg().size() == 1) { - List functionArgs = - ctx.functionArgs().functionArg(); - - String castExpresstion = - String.format("cast( %s as String)", functionArgs.getFirst().getText()); - DataTypeFunctionCallContext toStringDataTypeConversionContext = - this.createDataTypeFunctionCallContext(castExpresstion); - return new Cast( - visit(toStringDataTypeConversionContext.logicalExpression()), - visit(toStringDataTypeConversionContext.convertedDataType())); - // - } else { - return buildFunction(mappedName, ctx.functionArgs().functionArg()); - } - } else if (mappedName != null && mappedName.equals("tonumber")) { - - return buildFunction(mappedName, ctx.functionArgs().functionArg()); - - } else { - return new Cast(visit(ctx.logicalExpression()), visit(ctx.convertedDataType())); - } - } else { return new Cast(visit(ctx.logicalExpression()), visit(ctx.convertedDataType())); - } } @Override From a2accf5bc8caa7ad804fba592cc15954dc8fbd28 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Oct 2025 21:33:51 -0700 Subject: [PATCH 17/33] updated recommended changes Signed-off-by: Asif Bashar --- .../function/PPLBuiltinOperators.java | 9 +- .../expression/function/PPLFuncImpTable.java | 226 +++++++++++++++++- 2 files changed, 233 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index 14f82947c74..7fd768a8d8b 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -60,7 +60,14 @@ import org.opensearch.sql.expression.function.jsonUDF.JsonFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonKeysFunctionImpl; import org.opensearch.sql.expression.function.jsonUDF.JsonSetFunctionImpl; -import org.opensearch.sql.expression.function.udf.*; +import org.opensearch.sql.expression.function.udf.CryptographicFunction; +import org.opensearch.sql.expression.function.udf.ParseFunction; +import org.opensearch.sql.expression.function.udf.RelevanceQueryFunction; +import org.opensearch.sql.expression.function.udf.RexExtractFunction; +import org.opensearch.sql.expression.function.udf.RexExtractMultiFunction; +import org.opensearch.sql.expression.function.udf.RexOffsetFunction; +import org.opensearch.sql.expression.function.udf.SpanFunction; +import org.opensearch.sql.expression.function.udf.ToNumberFunction; import org.opensearch.sql.expression.function.udf.condition.EarliestFunction; import org.opensearch.sql.expression.function.udf.condition.EnhancedCoalesceFunction; import org.opensearch.sql.expression.function.udf.condition.LatestFunction; diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index 6f61eb14b48..d9e594f089f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -8,7 +8,231 @@ import static org.apache.calcite.sql.SqlJsonConstructorNullClause.NULL_ON_NULL; import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.TYPE_FACTORY; import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.getLegacyTypeName; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.*; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ABS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ACOS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ADD; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ADDDATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ADDFUNCTION; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ADDTIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.AND; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ARRAY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ARRAY_LENGTH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ASCII; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ASIN; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ATAN; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ATAN2; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.AVG; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CBRT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CEIL; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CEILING; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CIDRMATCH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.COALESCE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CONCAT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CONCAT_WS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CONV; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CONVERT_TZ; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.COS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.COSH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.COT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.COUNT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CRC32; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CURDATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CURRENT_DATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CURRENT_TIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CURRENT_TIMESTAMP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.CURTIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DATEDIFF; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DATETIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DATE_ADD; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DATE_FORMAT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DATE_SUB; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DAY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DAYNAME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DAYOFMONTH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DAYOFWEEK; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DAYOFYEAR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DAY_OF_MONTH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DAY_OF_WEEK; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DAY_OF_YEAR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DEGREES; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DIVIDE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.DIVIDEFUNCTION; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.E; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.EARLIEST; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.EQUAL; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.EXISTS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.EXP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.EXPM1; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.EXTRACT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.FILTER; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.FIRST; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.FLOOR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.FORALL; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.FROM_DAYS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.FROM_UNIXTIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.GET_FORMAT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.GREATER; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.GTE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.HOUR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.HOUR_OF_DAY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.IF; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.IFNULL; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_GROK; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_ITEM; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_PARSE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_PATTERN; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_PATTERN_PARSER; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_REGEXP_REPLACE_3; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_REGEXP_REPLACE_5; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_REGEXP_REPLACE_PG_4; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_TRANSLATE3; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.IS_BLANK; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.IS_EMPTY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.IS_NOT_NULL; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.IS_NULL; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.IS_PRESENT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_APPEND; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_ARRAY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_ARRAY_LENGTH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_DELETE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_EXTEND; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_EXTRACT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_EXTRACT_ALL; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_KEYS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_OBJECT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_SET; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.JSON_VALID; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LAST; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LAST_DAY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LATEST; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LEFT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LENGTH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LESS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LIKE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LIST; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LN; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LOCALTIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LOCALTIMESTAMP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LOCATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LOG; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LOG10; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LOG2; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LOWER; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LTE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.LTRIM; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MAKEDATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MAKETIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MAP_APPEND; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MAP_CONCAT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MAP_REMOVE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MATCH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MATCH_BOOL_PREFIX; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MATCH_PHRASE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MATCH_PHRASE_PREFIX; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MAX; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MD5; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MEDIAN; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MICROSECOND; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MIN; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MINUTE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MINUTE_OF_DAY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MINUTE_OF_HOUR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MOD; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MODULUS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MODULUSFUNCTION; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MONTH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MONTHNAME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MONTH_OF_YEAR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTIPLY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTIPLYFUNCTION; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTI_MATCH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MVAPPEND; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.MVJOIN; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOTEQUAL; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOW; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.NULLIF; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.OR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.PERCENTILE_APPROX; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.PERIOD_ADD; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.PERIOD_DIFF; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.PI; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.POSITION; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.POW; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.POWER; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.QUARTER; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.QUERY_STRING; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.RADIANS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.RAND; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.REDUCE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.REGEXP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.REGEX_MATCH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.REPLACE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.REVERSE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.REX_EXTRACT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.REX_EXTRACT_MULTI; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.REX_OFFSET; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.RIGHT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.RINT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.ROUND; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.RTRIM; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SECOND; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SECOND_OF_MINUTE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SEC_TO_TIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SHA1; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SHA2; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SIGN; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SIGNUM; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SIMPLE_QUERY_STRING; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SIN; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SINH; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SPAN; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SQRT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.STDDEV_POP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.STDDEV_SAMP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.STRCMP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.STRFTIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.STR_TO_DATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUBDATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUBSTR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUBSTRING; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUBTIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUBTRACT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUBTRACTFUNCTION; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SUM; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.SYSDATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TAKE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIMEDIFF; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIMESTAMP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIMESTAMPADD; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIMESTAMPDIFF; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIME_FORMAT; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIME_TO_SEC; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TO_DAYS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TO_SECONDS; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TRANSFORM; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TRIM; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TRUNCATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TYPEOF; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.UNIX_TIMESTAMP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.UPPER; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.UTC_DATE; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.UTC_TIME; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.UTC_TIMESTAMP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.VALUES; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.VARPOP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.VARSAMP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.WEEK; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.WEEKDAY; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.WEEKOFYEAR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.WEEK_OF_YEAR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.XOR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.YEAR; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.YEARWEEK; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TONUMBER; import com.google.common.collect.ImmutableMap; import java.math.BigDecimal; From 3adda5d37467116c9e8b1591832755281361ce9e Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Oct 2025 21:36:28 -0700 Subject: [PATCH 18/33] updated recommended changes Signed-off-by: Asif Bashar --- .../sql/ppl/parser/AstExpressionBuilder.java | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index b70226d039a..f7f6b3e914a 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -414,24 +414,10 @@ private Function buildFunction( functionName, args.stream().map(this::visitFunctionArg).collect(Collectors.toList())); } - public DataTypeFunctionCallContext createDataTypeFunctionCallContext(String castExpression) { - // Create a case-insensitive character stream from the input - CaseInsensitiveCharStream charStream = new CaseInsensitiveCharStream(castExpression); - - // Create lexer and parser - OpenSearchPPLLexer lexer = new OpenSearchPPLLexer(charStream); - CommonTokenStream tokens = new CommonTokenStream(lexer); - OpenSearchPPLParser parser = new OpenSearchPPLParser(tokens); - - // Parse the expression - cast is part of evalFunctionCall - DataTypeFunctionCallContext evalContext = parser.dataTypeFunctionCall(); - return evalContext; - } - /** Cast function. */ @Override public UnresolvedExpression visitDataTypeFunctionCall(DataTypeFunctionCallContext ctx) { - return new Cast(visit(ctx.logicalExpression()), visit(ctx.convertedDataType())); + return new Cast(visit(ctx.logicalExpression()), visit(ctx.convertedDataType())); } @Override From 17b1d86c3098b1fa11171ffe05fdcaa47fbf53d8 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Oct 2025 21:38:11 -0700 Subject: [PATCH 19/33] updated recommended changes Signed-off-by: Asif Bashar --- .../org/opensearch/sql/ppl/parser/AstExpressionBuilder.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index f7f6b3e914a..f037376f5c2 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -19,7 +19,6 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; -import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.RuleContext; import org.antlr.v4.runtime.tree.ParseTree; @@ -30,10 +29,8 @@ import org.opensearch.sql.ast.expression.subquery.ScalarSubquery; import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.calcite.plan.OpenSearchConstants; -import org.opensearch.sql.common.antlr.CaseInsensitiveCharStream; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.common.utils.StringUtils; -import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLLexer; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.BinaryArithmeticContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.BooleanLiteralContext; From eca7d139fce7d750e2be988facd8764dd6d1674c Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Oct 2025 21:38:59 -0700 Subject: [PATCH 20/33] updated recommended changes Signed-off-by: Asif Bashar --- .../opensearch/sql/expression/function/BuiltinFunctionName.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index fe862adc322..df3ae06d5c8 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -149,6 +149,7 @@ public enum BuiltinFunctionName { SYSDATE(FunctionName.of("sysdate")), /** Text Functions. */ + TOSTRING(FunctionName.of("tostring")), TONUMBER(FunctionName.of("tonumber")), /** IP Functions. */ From e780c156df19215bc03e4ba45988b32393f6ddcc Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Oct 2025 21:40:50 -0700 Subject: [PATCH 21/33] updated recommended changes Signed-off-by: Asif Bashar --- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 2 -- 1 file changed, 2 deletions(-) diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index a26b7381797..d386813fc96 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -847,13 +847,11 @@ evalFunctionCall : evalFunctionName LT_PRTHS functionArgs RT_PRTHS ; - // cast function dataTypeFunctionCall : CAST LT_PRTHS logicalExpression AS convertedDataType RT_PRTHS ; - convertedDataType : typeName = DATE | typeName = TIME From e3884b0a5ac00d147268fe964dcf554739685c88 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 21 Oct 2025 21:42:31 -0700 Subject: [PATCH 22/33] updated recommended changes Signed-off-by: Asif Bashar --- .../org/opensearch/sql/expression/function/PPLFuncImpTable.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index d9e594f089f..a5d0d1b8712 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -1062,7 +1062,6 @@ void populate() { SqlTypeFamily.INTEGER, SqlTypeFamily.INTEGER)), false)); - register( LOG, (FunctionImp2) From 042d3e821a03a2c5a5438f449bfbd983d8e53057 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Wed, 22 Oct 2025 10:11:40 -0700 Subject: [PATCH 23/33] spotless apply Signed-off-by: Asif Bashar --- .../sql/calcite/utils/PPLOperandTypes.java | 10 +- .../expression/function/PPLFuncImpTable.java | 2 +- .../function/udf/ToNumberFunction.java | 179 ++++++++---------- .../function/udf/ToNumberFunctionTest.java | 112 ++++++----- 4 files changed, 153 insertions(+), 150 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java index b00fa607d55..c64fa6ef4d5 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java @@ -106,12 +106,12 @@ private PPLOperandTypes() {} SqlTypeFamily.CHARACTER, SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER)); public static final UDFOperandMetadata STRING_OR_STRING_INTEGER = - UDFOperandMetadata.wrap( - (CompositeOperandTypeChecker) - OperandTypes.family(SqlTypeFamily.CHARACTER).or(OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER))); - + UDFOperandMetadata.wrap( + (CompositeOperandTypeChecker) + OperandTypes.family(SqlTypeFamily.CHARACTER) + .or(OperandTypes.family(SqlTypeFamily.CHARACTER, SqlTypeFamily.INTEGER))); - public static final UDFOperandMetadata STRING_STRING_INTEGER_INTEGER = + public static final UDFOperandMetadata STRING_STRING_INTEGER_INTEGER = UDFOperandMetadata.wrap( OperandTypes.family( SqlTypeFamily.CHARACTER, diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index a5d0d1b8712..4339619275c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -211,6 +211,7 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIMESTAMPDIFF; import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIME_FORMAT; import static org.opensearch.sql.expression.function.BuiltinFunctionName.TIME_TO_SEC; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.TONUMBER; import static org.opensearch.sql.expression.function.BuiltinFunctionName.TO_DAYS; import static org.opensearch.sql.expression.function.BuiltinFunctionName.TO_SECONDS; import static org.opensearch.sql.expression.function.BuiltinFunctionName.TRANSFORM; @@ -232,7 +233,6 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.XOR; import static org.opensearch.sql.expression.function.BuiltinFunctionName.YEAR; import static org.opensearch.sql.expression.function.BuiltinFunctionName.YEARWEEK; -import static org.opensearch.sql.expression.function.BuiltinFunctionName.TONUMBER; import com.google.common.collect.ImmutableMap; import java.math.BigDecimal; diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java index 95b37a0107c..0c5b576b297 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java @@ -5,12 +5,7 @@ package org.opensearch.sql.expression.function.udf; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.text.NumberFormat; import java.util.List; -import java.util.Locale; - import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; @@ -22,7 +17,6 @@ import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.calcite.utils.PPLOperandTypes; -import org.opensearch.sql.calcite.utils.PPLReturnTypes; import org.opensearch.sql.expression.function.ImplementorUDF; import org.opensearch.sql.expression.function.UDFOperandMetadata; @@ -36,105 +30,92 @@ * HH:MM:SS. if not format parameter provided, then consider value as boolean */ public class ToNumberFunction extends ImplementorUDF { - public ToNumberFunction() { - super( - new org.opensearch.sql.expression.function.udf.ToNumberFunction.ToNumberImplementor(), - NullPolicy.ANY); - } - - public static final String DURATION_FORMAT = "duration"; - public static final String DURATION_MILLIS_FORMAT = "duration_millis"; - public static final String HEX_FORMAT = "hex"; - public static final String COMMAS_FORMAT = "commas"; - public static final String BINARY_FORMAT = "binary"; - public static final SqlFunctions.DateFormatFunction dateTimeFormatter = - new SqlFunctions.DateFormatFunction(); - public static final String format24hour = "%H:%M:%S"; // 24-hour format + public ToNumberFunction() { + super( + new org.opensearch.sql.expression.function.udf.ToNumberFunction.ToNumberImplementor(), + NullPolicy.ANY); + } + + public static final String DURATION_FORMAT = "duration"; + public static final String DURATION_MILLIS_FORMAT = "duration_millis"; + public static final String HEX_FORMAT = "hex"; + public static final String COMMAS_FORMAT = "commas"; + public static final String BINARY_FORMAT = "binary"; + public static final SqlFunctions.DateFormatFunction dateTimeFormatter = + new SqlFunctions.DateFormatFunction(); + public static final String format24hour = "%H:%M:%S"; // 24-hour format + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return ReturnTypes.DOUBLE_FORCE_NULLABLE; + } + + @Override + public UDFOperandMetadata getOperandMetadata() { + return PPLOperandTypes.STRING_OR_STRING_INTEGER; + } + + public static class ToNumberImplementor implements NotNullImplementor { @Override - public SqlReturnTypeInference getReturnTypeInference() { - return ReturnTypes.DOUBLE_FORCE_NULLABLE; + public Expression implement( + RexToLixTranslator translator, RexCall call, List translatedOperands) { + Expression fieldValue = translatedOperands.get(0); + int base = 10; + if (translatedOperands.size() > 1) { + Expression baseExpr = translatedOperands.get(1); + return Expressions.call(ToNumberFunction.class, "toNumber", fieldValue, baseExpr); + } else { + return Expressions.call(ToNumberFunction.class, "toNumber", fieldValue); + } } + } - @Override - public UDFOperandMetadata getOperandMetadata() { - return PPLOperandTypes.STRING_OR_STRING_INTEGER; - } - - public static class ToNumberImplementor implements NotNullImplementor { - - @Override - public Expression implement( - RexToLixTranslator translator, RexCall call, List translatedOperands) { - Expression fieldValue = translatedOperands.get(0); - int base = 10; - if (translatedOperands.size() > 1) { - Expression baseExpr = translatedOperands.get(1); - return Expressions.call(ToNumberFunction.class, "toNumber", fieldValue, baseExpr); - } else { - return Expressions.call(ToNumberFunction.class, "toNumber", fieldValue); - } - - + @Strict + public static Number toNumber(String numStr) { + return toNumber(numStr, 10); + } - - - } + @Strict + public static Number toNumber(String numStr, int base) { + if (base < 2 || base > 36) { + throw new IllegalArgumentException("Base must be between 2 and 36"); } - @Strict - public static Number toNumber(String numStr) { - return toNumber(numStr, 10); + if (numStr.contains(".")) { + + boolean isNegative = numStr.startsWith("-"); + if (isNegative) { + numStr = numStr.substring(1); + } + + // Split integer and fractional parts + String[] parts = numStr.split("\\."); + String intPart = parts[0]; + String fracPart = parts.length > 1 ? parts[1] : ""; + + // Convert integer part + double intValue = 0; + for (char c : intPart.toCharArray()) { + int digit = Character.digit(c, base); + if (digit < 0) throw new IllegalArgumentException("Invalid digit: " + c); + intValue = intValue * base + digit; + } + + // Convert fractional part + double fracValue = 0; + double divisor = base; + for (char c : fracPart.toCharArray()) { + int digit = Character.digit(c, base); + if (digit < 0) throw new IllegalArgumentException("Invalid digit: " + c); + fracValue += (double) digit / divisor; + divisor *= base; + } + + double result = intValue + fracValue; + return isNegative ? -result : result; + } else { + return Integer.parseInt(numStr, base); } - - @Strict - public static Number toNumber(String numStr, int base) { - if (base < 2 || base > 36) { - throw new IllegalArgumentException("Base must be between 2 and 36"); - } - - if (numStr.contains(".")) { - - - boolean isNegative = numStr.startsWith("-"); - if (isNegative) { - numStr = numStr.substring(1); - } - - // Split integer and fractional parts - String[] parts = numStr.split("\\."); - String intPart = parts[0]; - String fracPart = parts.length > 1 ? parts[1] : ""; - - // Convert integer part - double intValue = 0; - for (char c : intPart.toCharArray()) { - int digit = Character.digit(c, base); - if (digit < 0) - throw new IllegalArgumentException("Invalid digit: " + c); - intValue = intValue * base + digit; - } - - // Convert fractional part - double fracValue = 0; - double divisor = base; - for (char c : fracPart.toCharArray()) { - int digit = Character.digit(c, base); - if (digit < 0) - throw new IllegalArgumentException("Invalid digit: " + c); - fracValue += (double) digit / divisor; - divisor *= base; - } - - double result = intValue + fracValue; - return isNegative ? -result : result; - } else { - return Integer.parseInt(numStr, base); - } - } - + } } - - - - diff --git a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java index d7f2b6fd685..7c29f75ba0a 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java @@ -7,8 +7,6 @@ import static org.junit.jupiter.api.Assertions.*; -import java.math.BigDecimal; -import java.util.Locale; import org.apache.calcite.sql.type.ReturnTypes; import org.junit.jupiter.api.Test; import org.opensearch.sql.calcite.utils.PPLOperandTypes; @@ -116,55 +114,77 @@ void testToNumberWithZeroIntegerPart() { @Test void testToNumberInvalidBase() { - assertThrows(IllegalArgumentException.class, () -> { - ToNumberFunction.toNumber("123", 1); - }); - - assertThrows(IllegalArgumentException.class, () -> { - ToNumberFunction.toNumber("123", 37); - }); - - assertThrows(IllegalArgumentException.class, () -> { - ToNumberFunction.toNumber("123", 0); - }); - - assertThrows(IllegalArgumentException.class, () -> { - ToNumberFunction.toNumber("123", -1); - }); + assertThrows( + IllegalArgumentException.class, + () -> { + ToNumberFunction.toNumber("123", 1); + }); + + assertThrows( + IllegalArgumentException.class, + () -> { + ToNumberFunction.toNumber("123", 37); + }); + + assertThrows( + IllegalArgumentException.class, + () -> { + ToNumberFunction.toNumber("123", 0); + }); + + assertThrows( + IllegalArgumentException.class, + () -> { + ToNumberFunction.toNumber("123", -1); + }); } @Test void testToNumberInvalidDigits() { - assertThrows(IllegalArgumentException.class, () -> { - ToNumberFunction.toNumber("12A", 10); - }); - - assertThrows(IllegalArgumentException.class, () -> { - ToNumberFunction.toNumber("102", 2); - }); - - assertThrows(IllegalArgumentException.class, () -> { - ToNumberFunction.toNumber("189", 8); - }); - - assertThrows(IllegalArgumentException.class, () -> { - ToNumberFunction.toNumber("GHI", 16); - }); + assertThrows( + IllegalArgumentException.class, + () -> { + ToNumberFunction.toNumber("12A", 10); + }); + + assertThrows( + IllegalArgumentException.class, + () -> { + ToNumberFunction.toNumber("102", 2); + }); + + assertThrows( + IllegalArgumentException.class, + () -> { + ToNumberFunction.toNumber("189", 8); + }); + + assertThrows( + IllegalArgumentException.class, + () -> { + ToNumberFunction.toNumber("GHI", 16); + }); } @Test void testToNumberInvalidFractionalDigits() { - assertThrows(IllegalArgumentException.class, () -> { - ToNumberFunction.toNumber("10.2", 2); - }); - - assertThrows(IllegalArgumentException.class, () -> { - ToNumberFunction.toNumber("FF.G", 16); - }); - - assertThrows(IllegalArgumentException.class, () -> { - ToNumberFunction.toNumber("123.ABC", 10); - }); + assertThrows( + IllegalArgumentException.class, + () -> { + ToNumberFunction.toNumber("10.2", 2); + }); + + assertThrows( + IllegalArgumentException.class, + () -> { + ToNumberFunction.toNumber("FF.G", 16); + }); + + assertThrows( + IllegalArgumentException.class, + () -> { + ToNumberFunction.toNumber("123.ABC", 10); + }); } @Test @@ -177,8 +197,10 @@ void testToNumberEdgeCases() { @Test void testToNumberLargeNumbers() { - assertEquals(Integer.MAX_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MAX_VALUE), 10)); - assertEquals(Integer.MIN_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MIN_VALUE), 10)); + assertEquals( + Integer.MAX_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MAX_VALUE), 10)); + assertEquals( + Integer.MIN_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MIN_VALUE), 10)); } @Test From 58d8e1b5714d9f02f80b910da847741cc309f713 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 28 Oct 2025 10:53:26 -0700 Subject: [PATCH 24/33] merge conflict fix Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 51 ++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 849d2334e41..ffe98228c39 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -117,3 +117,54 @@ Use string in comparison operator example :: | True | False | True | False | True | True | null | +------+-------+------+-------+------+------+------+ + + +TONUMBER +----------- + +Description +>>>>>>>>>>> + +The following usage options are available, depending on the parameter types and the number of parameters. + +Usage: tonumber(string, [base]) converts the value in first argument to provided base type string in second argument. If second argument is not provided, then it converts to base 10 number representation. + +Return type: Number + + +You can use this function with the eval commands and as part of eval expressions. +Base values can be between 2 and 36. + +You can use this function to convert a string representation of a binary number to return the corresponding number in base 10. + +Following example converts a string in binary to the number representation:: + + os> source=EMP | eval int_value = tonumber('010101',2) | fields int_value | head 1 + fetched rows / total rows = 1/1 + +---------------+ + | int_value | + |---------------+ + | 21.0 | + +---------------+ + + +Following example converts a string in hex to the number representation:: + + + os> source=EMP | eval int_value = tonumber('FA34',16) | fields int_value | head 1 + fetched rows / total rows = 1/1 + +---------------+ + | int_value | + |---------------+ + | 64052.0 | + +---------------+ + +Following example converts a string in decimal to the number representation:: + + os> source=EMP | eval int_value = tonumber('4598') | fields int_value | head 1 + fetched rows / total rows = 1/1 + +---------------+ + | int_value | + |---------------+ + | 4598.0 | + +---------------+ From 34f86355d4da09a88b761ac3e1190f7a609b1156 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Sat, 1 Nov 2025 22:48:31 -0700 Subject: [PATCH 25/33] added recommended changes Signed-off-by: Asif Bashar --- .../function/udf/ToNumberFunction.java | 24 ++-- .../function/udf/ToNumberFunctionTest.java | 105 +++++++----------- docs/user/ppl/functions/conversion.rst | 6 +- .../CalcitePPLToNumberFunctionTest.java | 71 ++++++++++++ 4 files changed, 133 insertions(+), 73 deletions(-) create mode 100644 ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java index 0c5b576b297..38f1734a8aa 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java @@ -104,18 +104,26 @@ public static Number toNumber(String numStr, int base) { // Convert fractional part double fracValue = 0; - double divisor = base; - for (char c : fracPart.toCharArray()) { - int digit = Character.digit(c, base); - if (digit < 0) throw new IllegalArgumentException("Invalid digit: " + c); - fracValue += (double) digit / divisor; - divisor *= base; + if (base == 10) { + double divisor = base; + for (char c : fracPart.toCharArray()) { + int digit = Character.digit(c, base); + if (digit < 0) throw new IllegalArgumentException("Invalid digit: " + c); + fracValue += (double) digit / divisor; + divisor *= base; + } } double result = intValue + fracValue; - return isNegative ? -result : result; + result = isNegative ? -result : result; + if (base == 10) { + return result; + } + else { + return (long) result; + } } else { - return Integer.parseInt(numStr, base); + return Long.parseLong(numStr, base); } } } diff --git a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java index 7c29f75ba0a..3a8c3185240 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java @@ -27,9 +27,9 @@ void testGetOperandMetadata() { @Test void testToNumberWithDefaultBase() { - assertEquals(123, ToNumberFunction.toNumber("123")); - assertEquals(0, ToNumberFunction.toNumber("0")); - assertEquals(-456, ToNumberFunction.toNumber("-456")); + assertEquals(123L, ToNumberFunction.toNumber("123")); + assertEquals(0L, ToNumberFunction.toNumber("0")); + assertEquals(-456L, ToNumberFunction.toNumber("-456")); assertEquals(123.45, ToNumberFunction.toNumber("123.45")); assertEquals(-123.45, ToNumberFunction.toNumber("-123.45")); assertEquals(0.5, ToNumberFunction.toNumber("0.5")); @@ -38,78 +38,78 @@ void testToNumberWithDefaultBase() { @Test void testToNumberWithBase10() { - assertEquals(123, ToNumberFunction.toNumber("123", 10)); - assertEquals(0, ToNumberFunction.toNumber("0", 10)); - assertEquals(-456, ToNumberFunction.toNumber("-456", 10)); + assertEquals(123L, ToNumberFunction.toNumber("123", 10)); + assertEquals(0L, ToNumberFunction.toNumber("0", 10)); + assertEquals(-456L, ToNumberFunction.toNumber("-456", 10)); assertEquals(123.45, ToNumberFunction.toNumber("123.45", 10)); assertEquals(-123.45, ToNumberFunction.toNumber("-123.45", 10)); } @Test void testToNumberWithBase2() { - assertEquals(5, ToNumberFunction.toNumber("101", 2)); - assertEquals(0, ToNumberFunction.toNumber("0", 2)); - assertEquals(1, ToNumberFunction.toNumber("1", 2)); - assertEquals(7, ToNumberFunction.toNumber("111", 2)); - assertEquals(10, ToNumberFunction.toNumber("1010", 2)); + assertEquals(5L, ToNumberFunction.toNumber("101", 2)); + assertEquals(0L, ToNumberFunction.toNumber("0", 2)); + assertEquals(1L, ToNumberFunction.toNumber("1", 2)); + assertEquals(7L, ToNumberFunction.toNumber("111", 2)); + assertEquals(10L, ToNumberFunction.toNumber("1010", 2)); } @Test void testToNumberWithBase8() { - assertEquals(64, ToNumberFunction.toNumber("100", 8)); - assertEquals(8, ToNumberFunction.toNumber("10", 8)); - assertEquals(83, ToNumberFunction.toNumber("123", 8)); - assertEquals(511, ToNumberFunction.toNumber("777", 8)); + assertEquals(64L, ToNumberFunction.toNumber("100", 8)); + assertEquals(8L, ToNumberFunction.toNumber("10", 8)); + assertEquals(83L, ToNumberFunction.toNumber("123", 8)); + assertEquals(511L, ToNumberFunction.toNumber("777", 8)); } @Test void testToNumberWithBase16() { - assertEquals(255, ToNumberFunction.toNumber("FF", 16)); - assertEquals(16, ToNumberFunction.toNumber("10", 16)); - assertEquals(171, ToNumberFunction.toNumber("AB", 16)); - assertEquals(291, ToNumberFunction.toNumber("123", 16)); - assertEquals(4095, ToNumberFunction.toNumber("FFF", 16)); + assertEquals(255L, ToNumberFunction.toNumber("FF", 16)); + assertEquals(16L, ToNumberFunction.toNumber("10", 16)); + assertEquals(171L, ToNumberFunction.toNumber("AB", 16)); + assertEquals(291L, ToNumberFunction.toNumber("123", 16)); + assertEquals(4095L, ToNumberFunction.toNumber("FFF", 16)); } @Test void testToNumberWithBase36() { - assertEquals(35, ToNumberFunction.toNumber("Z", 36)); - assertEquals(1295, ToNumberFunction.toNumber("ZZ", 36)); - assertEquals(46655, ToNumberFunction.toNumber("ZZZ", 36)); + assertEquals(35L, ToNumberFunction.toNumber("Z", 36)); + assertEquals(1295L, ToNumberFunction.toNumber("ZZ", 36)); + assertEquals(46655L, ToNumberFunction.toNumber("ZZZ", 36)); } @Test void testToNumberWithDecimalBase2() { - assertEquals(2.5, ToNumberFunction.toNumber("10.1", 2)); - assertEquals(1.5, ToNumberFunction.toNumber("1.1", 2)); - assertEquals(3.75, ToNumberFunction.toNumber("11.11", 2)); + assertEquals(2L, ToNumberFunction.toNumber("10.1", 2)); + assertEquals(1L, ToNumberFunction.toNumber("1.1", 2)); + assertEquals(3L, ToNumberFunction.toNumber("11.11", 2)); } @Test void testToNumberWithDecimalBase16() { - assertEquals(255.5, ToNumberFunction.toNumber("FF.8", 16)); - assertEquals(16.25, ToNumberFunction.toNumber("10.4", 16)); - assertEquals(171.6875, ToNumberFunction.toNumber("AB.B", 16)); + assertEquals(255L, ToNumberFunction.toNumber("FF.8", 16)); + assertEquals(16L, ToNumberFunction.toNumber("10.4", 16)); + assertEquals(171L, ToNumberFunction.toNumber("AB.B", 16)); } @Test void testToNumberWithNegativeDecimal() { - assertEquals(-2.5, ToNumberFunction.toNumber("-10.1", 2)); - assertEquals(-255.5, ToNumberFunction.toNumber("-FF.8", 16)); + assertEquals(-2L, ToNumberFunction.toNumber("-10.1", 2)); + assertEquals(-255L, ToNumberFunction.toNumber("-FF.8", 16)); assertEquals(-123.45, ToNumberFunction.toNumber("-123.45", 10)); } @Test void testToNumberWithEmptyFractionalPart() { assertEquals(123.0, ToNumberFunction.toNumber("123.", 10)); - assertEquals(255.0, ToNumberFunction.toNumber("FF.", 16)); - assertEquals(5.0, ToNumberFunction.toNumber("101.", 2)); + assertEquals(255L, ToNumberFunction.toNumber("FF.", 16)); + assertEquals(5L, ToNumberFunction.toNumber("101.", 2)); } @Test void testToNumberWithZeroIntegerPart() { assertEquals(0.5, ToNumberFunction.toNumber("0.5", 10)); - assertEquals(0.5, ToNumberFunction.toNumber("0.1", 2)); + assertEquals(0L, ToNumberFunction.toNumber("0.1", 2)); } @Test @@ -166,31 +166,12 @@ void testToNumberInvalidDigits() { }); } - @Test - void testToNumberInvalidFractionalDigits() { - assertThrows( - IllegalArgumentException.class, - () -> { - ToNumberFunction.toNumber("10.2", 2); - }); - assertThrows( - IllegalArgumentException.class, - () -> { - ToNumberFunction.toNumber("FF.G", 16); - }); - - assertThrows( - IllegalArgumentException.class, - () -> { - ToNumberFunction.toNumber("123.ABC", 10); - }); - } @Test void testToNumberEdgeCases() { - assertEquals(0, ToNumberFunction.toNumber("0", 2)); - assertEquals(0, ToNumberFunction.toNumber("0", 36)); + assertEquals(0L, ToNumberFunction.toNumber("0", 2)); + assertEquals(0L, ToNumberFunction.toNumber("0", 36)); assertEquals(0.0, ToNumberFunction.toNumber("0.0", 10)); assertEquals(0.0, ToNumberFunction.toNumber("0.000", 10)); } @@ -198,17 +179,17 @@ void testToNumberEdgeCases() { @Test void testToNumberLargeNumbers() { assertEquals( - Integer.MAX_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MAX_VALUE), 10)); + (long) Integer.MAX_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MAX_VALUE), 10)); assertEquals( - Integer.MIN_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MIN_VALUE), 10)); + (long) Integer.MIN_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MIN_VALUE), 10)); } @Test void testToNumberCaseInsensitivity() { - assertEquals(255, ToNumberFunction.toNumber("ff", 16)); - assertEquals(255, ToNumberFunction.toNumber("FF", 16)); - assertEquals(255, ToNumberFunction.toNumber("fF", 16)); - assertEquals(171, ToNumberFunction.toNumber("ab", 16)); - assertEquals(171, ToNumberFunction.toNumber("AB", 16)); + assertEquals(255L, ToNumberFunction.toNumber("ff", 16)); + assertEquals(255L, ToNumberFunction.toNumber("FF", 16)); + assertEquals(255L, ToNumberFunction.toNumber("fF", 16)); + assertEquals(171L, ToNumberFunction.toNumber("ab", 16)); + assertEquals(171L, ToNumberFunction.toNumber("AB", 16)); } } diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index ffe98228c39..0176c6a6003 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -139,7 +139,7 @@ You can use this function to convert a string representation of a binary number Following example converts a string in binary to the number representation:: - os> source=EMP | eval int_value = tonumber('010101',2) | fields int_value | head 1 + os> source=people | eval int_value = tonumber('010101',2) | fields int_value | head 1 fetched rows / total rows = 1/1 +---------------+ | int_value | @@ -151,7 +151,7 @@ Following example converts a string in binary to the number representation:: Following example converts a string in hex to the number representation:: - os> source=EMP | eval int_value = tonumber('FA34',16) | fields int_value | head 1 + os> source=people | eval int_value = tonumber('FA34',16) | fields int_value | head 1 fetched rows / total rows = 1/1 +---------------+ | int_value | @@ -161,7 +161,7 @@ Following example converts a string in hex to the number representation:: Following example converts a string in decimal to the number representation:: - os> source=EMP | eval int_value = tonumber('4598') | fields int_value | head 1 + os> source=people | eval int_value = tonumber('4598') | fields int_value | head 1 fetched rows / total rows = 1/1 +---------------+ | int_value | diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java new file mode 100644 index 00000000000..885812a9003 --- /dev/null +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java @@ -0,0 +1,71 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.calcite; + +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.test.CalciteAssert; +import org.junit.Test; + +public class CalcitePPLToNumberFunctionTest extends CalcitePPLAbstractTest { + + public CalcitePPLToNumberFunctionTest() { + super(CalciteAssert.SchemaSpec.SCOTT_WITH_TEMPORAL); + } + + @Test + public void testNumberBinary() { + String ppl = "source=EMP | eval int_value = tonumber('010101',2) | fields int_value|head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n LogicalProject(int_value=[TONUMBER('010101':VARCHAR, 2)])\n LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "int_value=21.0\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = "SELECT `TONUMBER`('010101', 2) `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + + @Test + public void testNumberHex() { + String ppl = "source=EMP | eval int_value = tonumber('FA34',16) | fields int_value|head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = "LogicalSort(fetch=[1])\n LogicalProject(int_value=[TONUMBER('FA34':VARCHAR, 16)])\n LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "int_value=64052.0\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = "SELECT `TONUMBER`('FA34', 16) `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testNumber() { + String ppl = "source=EMP | eval int_value = tonumber('4598') | fields int_value|head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = "LogicalSort(fetch=[1])\n LogicalProject(int_value=[TONUMBER('4598':VARCHAR)])\n LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "int_value=4598.0\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = "SELECT `TONUMBER`('4598') `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testNumberDecimal() { + String ppl = "source=EMP | eval int_value = tonumber('4598.54922') | fields int_value|head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = "LogicalSort(fetch=[1])\n LogicalProject(int_value=[TONUMBER('4598.54922':VARCHAR)])\n LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "int_value=4598.54922\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = "SELECT `TONUMBER`('4598.54922') `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } +} From 8279bf3c18889272ee8ac0266a5fdeaa3c4ab5b2 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 4 Nov 2025 18:13:10 -0800 Subject: [PATCH 26/33] fixed doctest for conversion.rst and applied spotless Signed-off-by: Asif Bashar --- .../function/udf/ToNumberFunction.java | 23 +++-- .../function/udf/ToNumberFunctionTest.java | 6 +- docs/user/ppl/functions/conversion.rst | 78 ++++++++--------- .../CalcitePPLToNumberFunctionTest.java | 85 +++++++++++-------- 4 files changed, 101 insertions(+), 91 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java index 38f1734a8aa..479cbefaf87 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java @@ -105,22 +105,21 @@ public static Number toNumber(String numStr, int base) { // Convert fractional part double fracValue = 0; if (base == 10) { - double divisor = base; - for (char c : fracPart.toCharArray()) { - int digit = Character.digit(c, base); - if (digit < 0) throw new IllegalArgumentException("Invalid digit: " + c); - fracValue += (double) digit / divisor; - divisor *= base; - } + double divisor = base; + for (char c : fracPart.toCharArray()) { + int digit = Character.digit(c, base); + if (digit < 0) throw new IllegalArgumentException("Invalid digit: " + c); + fracValue += (double) digit / divisor; + divisor *= base; + } } double result = intValue + fracValue; - result = isNegative ? -result : result; + result = isNegative ? -result : result; if (base == 10) { - return result; - } - else { - return (long) result; + return result; + } else { + return (long) result; } } else { return Long.parseLong(numStr, base); diff --git a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java index 3a8c3185240..cd976956408 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java @@ -166,8 +166,6 @@ void testToNumberInvalidDigits() { }); } - - @Test void testToNumberEdgeCases() { assertEquals(0L, ToNumberFunction.toNumber("0", 2)); @@ -179,9 +177,9 @@ void testToNumberEdgeCases() { @Test void testToNumberLargeNumbers() { assertEquals( - (long) Integer.MAX_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MAX_VALUE), 10)); + (long) Integer.MAX_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MAX_VALUE), 10)); assertEquals( - (long) Integer.MIN_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MIN_VALUE), 10)); + (long) Integer.MIN_VALUE, ToNumberFunction.toNumber(String.valueOf(Integer.MIN_VALUE), 10)); } @Test diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index 0176c6a6003..fd34681b03f 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -43,21 +43,21 @@ Cast to string example:: os> source=people | eval `cbool` = CAST(true as string), `cint` = CAST(1 as string), `cdate` = CAST(CAST('2012-08-07' as date) as string) | fields `cbool`, `cint`, `cdate` fetched rows / total rows = 1/1 - +-------+------+------------+ - | cbool | cint | cdate | - |-------+------+------------| - | TRUE | 1 | 2012-08-07 | - +-------+------+------------+ + +---------+--------+------------+ + | cbool | cint | cdate | + |---------+--------+------------| + | TRUE | 1 | 2012-08-07 | + +---------+--------+------------+ Cast to number example:: os> source=people | eval `cbool` = CAST(true as int), `cstring` = CAST('1' as int) | fields `cbool`, `cstring` fetched rows / total rows = 1/1 - +-------+---------+ - | cbool | cstring | - |-------+---------| - | 1 | 1 | - +-------+---------+ + +---------+-----------+ + | cbool | cstring | + |---------+-----------| + | 1 | 1 | + +---------+-----------+ Cast to date example:: @@ -73,11 +73,11 @@ Cast function can be chained:: os> source=people | eval `cbool` = CAST(CAST(true as string) as boolean) | fields `cbool` fetched rows / total rows = 1/1 - +-------+ - | cbool | - |-------| - | True | - +-------+ + +---------+ + | cbool | + |---------| + | True | + +---------+ IMPLICIT (AUTO) TYPE CONVERSION @@ -101,11 +101,11 @@ Use string in arithmetic operator example :: os> source=people | eval divide="5"/10, multiply="5" * 10, add="5" + 10, minus="5" - 10, concat="5" + "5" | fields divide, multiply, add, minus, concat fetched rows / total rows = 1/1 - +--------+----------+------+-------+--------+ - | divide | multiply | add | minus | concat | - |--------+----------+------+-------+--------| - | 0.5 | 50.0 | 15.0 | -5.0 | 55 | - +--------+----------+------+-------+--------+ + +----------+------------+-------+---------+----------+ + | divide | multiply | add | minus | concat | + |----------+------------+-------+---------+----------| + | 0.5 | 50.0 | 15.0 | -5.0 | 55 | + +----------+------------+-------+---------+----------+ Use string in comparison operator example :: @@ -120,7 +120,7 @@ Use string in comparison operator example :: TONUMBER ------------ +-------- Description >>>>>>>>>>> @@ -141,30 +141,30 @@ Following example converts a string in binary to the number representation:: os> source=people | eval int_value = tonumber('010101',2) | fields int_value | head 1 fetched rows / total rows = 1/1 - +---------------+ - | int_value | - |---------------+ - | 21.0 | - +---------------+ + +-------------+ + | int_value | + |-------------| + | 21.0 | + +-------------+ Following example converts a string in hex to the number representation:: - os> source=people | eval int_value = tonumber('FA34',16) | fields int_value | head 1 fetched rows / total rows = 1/1 - +---------------+ - | int_value | - |---------------+ - | 64052.0 | - +---------------+ + +-------------+ + | int_value | + |-------------| + | 64052.0 | + +-------------+ Following example converts a string in decimal to the number representation:: - os> source=people | eval int_value = tonumber('4598') | fields int_value | head 1 - fetched rows / total rows = 1/1 - +---------------+ - | int_value | - |---------------+ - | 4598.0 | - +---------------+ + os> source=people | eval int_value = tonumber('4598') | fields int_value | head 1 + fetched rows / total rows = 1/1 + +-------------+ + | int_value | + |-------------| + | 4598.0 | + +-------------+ + diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java index 885812a9003..37053e59f51 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java @@ -20,52 +20,65 @@ public void testNumberBinary() { String ppl = "source=EMP | eval int_value = tonumber('010101',2) | fields int_value|head 1"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalSort(fetch=[1])\n LogicalProject(int_value=[TONUMBER('010101':VARCHAR, 2)])\n LogicalTableScan(table=[[scott, EMP]])\n"; + "LogicalSort(fetch=[1])\n" + + " LogicalProject(int_value=[TONUMBER('010101':VARCHAR, 2)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedResult = "int_value=21.0\n"; verifyResult(root, expectedResult); - String expectedSparkSql = "SELECT `TONUMBER`('010101', 2) `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + String expectedSparkSql = + "SELECT `TONUMBER`('010101', 2) `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; verifyPPLToSparkSQL(root, expectedSparkSql); } + @Test + public void testNumberHex() { + String ppl = "source=EMP | eval int_value = tonumber('FA34',16) | fields int_value|head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(int_value=[TONUMBER('FA34':VARCHAR, 16)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "int_value=64052.0\n"; + verifyResult(root, expectedResult); - @Test - public void testNumberHex() { - String ppl = "source=EMP | eval int_value = tonumber('FA34',16) | fields int_value|head 1"; - RelNode root = getRelNode(ppl); - String expectedLogical = "LogicalSort(fetch=[1])\n LogicalProject(int_value=[TONUMBER('FA34':VARCHAR, 16)])\n LogicalTableScan(table=[[scott, EMP]])\n"; - verifyLogical(root, expectedLogical); - String expectedResult = "int_value=64052.0\n"; - verifyResult(root, expectedResult); - - String expectedSparkSql = "SELECT `TONUMBER`('FA34', 16) `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; - verifyPPLToSparkSQL(root, expectedSparkSql); - } + String expectedSparkSql = + "SELECT `TONUMBER`('FA34', 16) `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } - @Test - public void testNumber() { - String ppl = "source=EMP | eval int_value = tonumber('4598') | fields int_value|head 1"; - RelNode root = getRelNode(ppl); - String expectedLogical = "LogicalSort(fetch=[1])\n LogicalProject(int_value=[TONUMBER('4598':VARCHAR)])\n LogicalTableScan(table=[[scott, EMP]])\n"; - verifyLogical(root, expectedLogical); - String expectedResult = "int_value=4598.0\n"; - verifyResult(root, expectedResult); + @Test + public void testNumber() { + String ppl = "source=EMP | eval int_value = tonumber('4598') | fields int_value|head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(int_value=[TONUMBER('4598':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "int_value=4598.0\n"; + verifyResult(root, expectedResult); - String expectedSparkSql = "SELECT `TONUMBER`('4598') `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; - verifyPPLToSparkSQL(root, expectedSparkSql); - } + String expectedSparkSql = "SELECT `TONUMBER`('4598') `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } - @Test - public void testNumberDecimal() { - String ppl = "source=EMP | eval int_value = tonumber('4598.54922') | fields int_value|head 1"; - RelNode root = getRelNode(ppl); - String expectedLogical = "LogicalSort(fetch=[1])\n LogicalProject(int_value=[TONUMBER('4598.54922':VARCHAR)])\n LogicalTableScan(table=[[scott, EMP]])\n"; - verifyLogical(root, expectedLogical); - String expectedResult = "int_value=4598.54922\n"; - verifyResult(root, expectedResult); + @Test + public void testNumberDecimal() { + String ppl = "source=EMP | eval int_value = tonumber('4598.54922') | fields int_value|head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(int_value=[TONUMBER('4598.54922':VARCHAR)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "int_value=4598.54922\n"; + verifyResult(root, expectedResult); - String expectedSparkSql = "SELECT `TONUMBER`('4598.54922') `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; - verifyPPLToSparkSQL(root, expectedSparkSql); - } + String expectedSparkSql = + "SELECT `TONUMBER`('4598.54922') `int_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } } From b5346a6dcfb385f040cc103f8ad480678dc0156a Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 4 Nov 2025 18:20:04 -0800 Subject: [PATCH 27/33] removed decimal point from hex Signed-off-by: Asif Bashar --- .../function/udf/ToNumberFunctionTest.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java index cd976956408..d457a8a36a9 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java @@ -80,36 +80,36 @@ void testToNumberWithBase36() { @Test void testToNumberWithDecimalBase2() { - assertEquals(2L, ToNumberFunction.toNumber("10.1", 2)); - assertEquals(1L, ToNumberFunction.toNumber("1.1", 2)); - assertEquals(3L, ToNumberFunction.toNumber("11.11", 2)); + assertEquals(2L, ToNumberFunction.toNumber("10", 2)); + assertEquals(1L, ToNumberFunction.toNumber("1", 2)); + assertEquals(3L, ToNumberFunction.toNumber("11", 2)); } @Test void testToNumberWithDecimalBase16() { - assertEquals(255L, ToNumberFunction.toNumber("FF.8", 16)); - assertEquals(16L, ToNumberFunction.toNumber("10.4", 16)); - assertEquals(171L, ToNumberFunction.toNumber("AB.B", 16)); + assertEquals(255L, ToNumberFunction.toNumber("FF", 16)); + assertEquals(16L, ToNumberFunction.toNumber("10", 16)); + assertEquals(171L, ToNumberFunction.toNumber("AB", 16)); } @Test void testToNumberWithNegativeDecimal() { - assertEquals(-2L, ToNumberFunction.toNumber("-10.1", 2)); - assertEquals(-255L, ToNumberFunction.toNumber("-FF.8", 16)); + assertEquals(-2L, ToNumberFunction.toNumber("-10", 2)); + assertEquals(-255L, ToNumberFunction.toNumber("-FF", 16)); assertEquals(-123.45, ToNumberFunction.toNumber("-123.45", 10)); } @Test void testToNumberWithEmptyFractionalPart() { assertEquals(123.0, ToNumberFunction.toNumber("123.", 10)); - assertEquals(255L, ToNumberFunction.toNumber("FF.", 16)); - assertEquals(5L, ToNumberFunction.toNumber("101.", 2)); + assertEquals(255L, ToNumberFunction.toNumber("FF", 16)); + assertEquals(5L, ToNumberFunction.toNumber("101", 2)); } @Test void testToNumberWithZeroIntegerPart() { assertEquals(0.5, ToNumberFunction.toNumber("0.5", 10)); - assertEquals(0L, ToNumberFunction.toNumber("0.1", 2)); + assertEquals(0L, ToNumberFunction.toNumber("0", 2)); } @Test From 7cf867b4fa7a1e170df483c3b84e992b62c771a0 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Wed, 5 Nov 2025 09:56:12 -0800 Subject: [PATCH 28/33] added doctest fixes Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 70 +++++++++++++------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index fd34681b03f..52eb04b8e63 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -43,21 +43,21 @@ Cast to string example:: os> source=people | eval `cbool` = CAST(true as string), `cint` = CAST(1 as string), `cdate` = CAST(CAST('2012-08-07' as date) as string) | fields `cbool`, `cint`, `cdate` fetched rows / total rows = 1/1 - +---------+--------+------------+ - | cbool | cint | cdate | - |---------+--------+------------| - | TRUE | 1 | 2012-08-07 | - +---------+--------+------------+ + +-------+------+------------+ + | cbool | cint | cdate | + |-------+------+------------| + | TRUE | 1 | 2012-08-07 | + +-------+------+------------+ Cast to number example:: os> source=people | eval `cbool` = CAST(true as int), `cstring` = CAST('1' as int) | fields `cbool`, `cstring` fetched rows / total rows = 1/1 - +---------+-----------+ - | cbool | cstring | - |---------+-----------| - | 1 | 1 | - +---------+-----------+ + +-------+---------+ + | cbool | cstring | + |-------+---------| + | 1 | 1 | + +-------+---------+ Cast to date example:: @@ -73,11 +73,11 @@ Cast function can be chained:: os> source=people | eval `cbool` = CAST(CAST(true as string) as boolean) | fields `cbool` fetched rows / total rows = 1/1 - +---------+ - | cbool | - |---------| - | True | - +---------+ + +-------+ + | cbool | + |-------| + | True | + +-------+ IMPLICIT (AUTO) TYPE CONVERSION @@ -101,11 +101,11 @@ Use string in arithmetic operator example :: os> source=people | eval divide="5"/10, multiply="5" * 10, add="5" + 10, minus="5" - 10, concat="5" + "5" | fields divide, multiply, add, minus, concat fetched rows / total rows = 1/1 - +----------+------------+-------+---------+----------+ - | divide | multiply | add | minus | concat | - |----------+------------+-------+---------+----------| - | 0.5 | 50.0 | 15.0 | -5.0 | 55 | - +----------+------------+-------+---------+----------+ + +--------+----------+------+-------+--------+ + | divide | multiply | add | minus | concat | + |--------+----------+------+-------+--------| + | 0.5 | 50.0 | 15.0 | -5.0 | 55 | + +--------+----------+------+-------+--------+ Use string in comparison operator example :: @@ -141,30 +141,30 @@ Following example converts a string in binary to the number representation:: os> source=people | eval int_value = tonumber('010101',2) | fields int_value | head 1 fetched rows / total rows = 1/1 - +-------------+ - | int_value | - |-------------| - | 21.0 | - +-------------+ + +-----------+ + | int_value | + |-----------| + | 21.0 | + +-----------+ Following example converts a string in hex to the number representation:: os> source=people | eval int_value = tonumber('FA34',16) | fields int_value | head 1 fetched rows / total rows = 1/1 - +-------------+ - | int_value | - |-------------| - | 64052.0 | - +-------------+ + +-----------+ + | int_value | + |-----------| + | 64052.0 | + +-----------+ Following example converts a string in decimal to the number representation:: os> source=people | eval int_value = tonumber('4598') | fields int_value | head 1 fetched rows / total rows = 1/1 - +-------------+ - | int_value | - |-------------| - | 4598.0 | - +-------------+ + +-----------+ + | int_value | + |-----------| + | 4598.0 | + +-----------+ From 46f010bb62eca84723683347d56cf31323f9ff5b Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Wed, 5 Nov 2025 23:11:47 -0800 Subject: [PATCH 29/33] removed unused variables Signed-off-by: Asif Bashar --- .../sql/expression/function/udf/ToNumberFunction.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java index 479cbefaf87..ba30a7a7d1d 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java @@ -36,14 +36,7 @@ public ToNumberFunction() { NullPolicy.ANY); } - public static final String DURATION_FORMAT = "duration"; - public static final String DURATION_MILLIS_FORMAT = "duration_millis"; - public static final String HEX_FORMAT = "hex"; - public static final String COMMAS_FORMAT = "commas"; - public static final String BINARY_FORMAT = "binary"; - public static final SqlFunctions.DateFormatFunction dateTimeFormatter = - new SqlFunctions.DateFormatFunction(); - public static final String format24hour = "%H:%M:%S"; // 24-hour format + @Override public SqlReturnTypeInference getReturnTypeInference() { From b7afa1713253d1adc49d343467cdda50d98869a4 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 6 Nov 2025 06:36:34 -0800 Subject: [PATCH 30/33] spotless Signed-off-by: Asif Bashar --- .../sql/expression/function/udf/ToNumberFunction.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java index ba30a7a7d1d..5b3a49fe549 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java @@ -13,7 +13,6 @@ import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; import org.apache.calcite.rex.RexCall; -import org.apache.calcite.runtime.SqlFunctions; import org.apache.calcite.sql.type.ReturnTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.opensearch.sql.calcite.utils.PPLOperandTypes; @@ -36,8 +35,6 @@ public ToNumberFunction() { NullPolicy.ANY); } - - @Override public SqlReturnTypeInference getReturnTypeInference() { return ReturnTypes.DOUBLE_FORCE_NULLABLE; From 0f0125a127778b29f74f69aeb16119d9d3716074 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 6 Nov 2025 10:24:54 -0800 Subject: [PATCH 31/33] made recommended changes to simply parse to number, null when malformed Signed-off-by: Asif Bashar --- .../function/udf/ToNumberFunction.java | 52 +++++-------------- .../function/udf/ToNumberFunctionTest.java | 29 +++-------- 2 files changed, 19 insertions(+), 62 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java index 5b3a49fe549..2557121b0c8 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/ToNumberFunction.java @@ -5,6 +5,7 @@ package org.opensearch.sql.expression.function.udf; +import java.math.BigInteger; import java.util.List; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; @@ -69,50 +70,23 @@ public static Number toNumber(String numStr) { @Strict public static Number toNumber(String numStr, int base) { if (base < 2 || base > 36) { - throw new IllegalArgumentException("Base must be between 2 and 36"); + throw new IllegalArgumentException("Base has to be between 2 and 36."); } - - if (numStr.contains(".")) { - - boolean isNegative = numStr.startsWith("-"); - if (isNegative) { - numStr = numStr.substring(1); - } - - // Split integer and fractional parts - String[] parts = numStr.split("\\."); - String intPart = parts[0]; - String fracPart = parts.length > 1 ? parts[1] : ""; - - // Convert integer part - double intValue = 0; - for (char c : intPart.toCharArray()) { - int digit = Character.digit(c, base); - if (digit < 0) throw new IllegalArgumentException("Invalid digit: " + c); - intValue = intValue * base + digit; - } - - // Convert fractional part - double fracValue = 0; + Number result = null; + try { if (base == 10) { - double divisor = base; - for (char c : fracPart.toCharArray()) { - int digit = Character.digit(c, base); - if (digit < 0) throw new IllegalArgumentException("Invalid digit: " + c); - fracValue += (double) digit / divisor; - divisor *= base; + if (numStr.contains(".")) { + result = Double.parseDouble(numStr); + } else { + result = Long.parseLong(numStr); } - } - - double result = intValue + fracValue; - result = isNegative ? -result : result; - if (base == 10) { - return result; } else { - return (long) result; + BigInteger bigInteger = new BigInteger(numStr, base); + result = bigInteger.longValue(); } - } else { - return Long.parseLong(numStr, base); + } catch (Exception e) { + } + return result; } } diff --git a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java index d457a8a36a9..34ed102ec5b 100644 --- a/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/function/udf/ToNumberFunctionTest.java @@ -141,29 +141,12 @@ void testToNumberInvalidBase() { @Test void testToNumberInvalidDigits() { - assertThrows( - IllegalArgumentException.class, - () -> { - ToNumberFunction.toNumber("12A", 10); - }); - - assertThrows( - IllegalArgumentException.class, - () -> { - ToNumberFunction.toNumber("102", 2); - }); - - assertThrows( - IllegalArgumentException.class, - () -> { - ToNumberFunction.toNumber("189", 8); - }); - - assertThrows( - IllegalArgumentException.class, - () -> { - ToNumberFunction.toNumber("GHI", 16); - }); + assertEquals(null, ToNumberFunction.toNumber("12A", 10)); + assertEquals(null, ToNumberFunction.toNumber("102", 2)); + assertEquals(null, ToNumberFunction.toNumber("101.101", 2)); + assertEquals(null, ToNumberFunction.toNumber("189", 8)); + assertEquals(null, ToNumberFunction.toNumber("GHI", 16)); + assertEquals(null, ToNumberFunction.toNumber("FF.8", 16)); } @Test From 804db79a7ad546937cece7a99eea0e84af32c66a Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Sat, 8 Nov 2025 16:51:14 -0800 Subject: [PATCH 32/33] hex max limit doc and unit tests Signed-off-by: Asif Bashar --- docs/user/ppl/functions/conversion.rst | 3 +- .../CalcitePPLToNumberFunctionTest.java | 80 +++++++++++++++++++ 2 files changed, 82 insertions(+), 1 deletion(-) diff --git a/docs/user/ppl/functions/conversion.rst b/docs/user/ppl/functions/conversion.rst index ca9297b1f0d..c55e868f2cd 100644 --- a/docs/user/ppl/functions/conversion.rst +++ b/docs/user/ppl/functions/conversion.rst @@ -133,7 +133,8 @@ Return type: Number You can use this function with the eval commands and as part of eval expressions. -Base values can be between 2 and 36. +Base values can be between 2 and 36. The maximum value supported for base 10 is +(2-2^-52)·2^1023 and minimum is -(2-2^-52)·2^1023. +The maximum for other supported bases is 2^63-1 (or 7FFFFFFFFFFFFFFF) and minimum is -2^63 (or -7FFFFFFFFFFFFFFF). You can use this function to convert a string representation of a binary number to return the corresponding number in base 10. diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java index 37053e59f51..448cddeefec 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java @@ -49,6 +49,86 @@ public void testNumberHex() { verifyPPLToSparkSQL(root, expectedSparkSql); } + + @Test + public void testNumberHexMinLimit() { + String ppl = + "source=EMP | eval long_value = tonumber('-7FFFFFFFFFFFFFFF',16) | fields long_value|head" + + " 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(long_value=[TONUMBER('-7FFFFFFFFFFFFFFF':VARCHAR, 16)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "long_value=-9.223372036854776E18\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `TONUMBER`('-7FFFFFFFFFFFFFFF', 16) `long_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testNumberHexMaxLimit() { + String ppl = + "source=EMP | eval long_value = tonumber('7FFFFFFFFFFFFFFF',16) | fields long_value|head" + + " 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(long_value=[TONUMBER('7FFFFFFFFFFFFFFF':VARCHAR, 16)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "long_value=9.223372036854776E18\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `TONUMBER`('7FFFFFFFFFFFFFFF', 16) `long_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testNumberHexOverNegativeMaxLimit() { + String ppl = + "source=EMP | eval long_value = tonumber('-FFFFFFFFFFFFFFFF',16) | fields long_value|head" + + " 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(long_value=[TONUMBER('-FFFFFFFFFFFFFFFF':VARCHAR, 16)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "long_value=1.0\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `TONUMBER`('-FFFFFFFFFFFFFFFF', 16) `long_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testNumberHexOverPositiveMaxLimit() { + String ppl = + "source=EMP | eval long_value = tonumber('FFFFFFFFFFFFFFFF',16) | fields long_value|head 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(long_value=[TONUMBER('FFFFFFFFFFFFFFFF':VARCHAR, 16)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "long_value=-1.0\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `TONUMBER`('FFFFFFFFFFFFFFFF', 16) `long_value`\nFROM `scott`.`EMP`\nLIMIT 1"; + + verifyPPLToSparkSQL(root, expectedSparkSql); + } + @Test public void testNumber() { String ppl = "source=EMP | eval int_value = tonumber('4598') | fields int_value|head 1"; From 75883979380e8cad8b0f4e2318ecaf748fd6d0aa Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Sun, 9 Nov 2025 10:51:54 -0800 Subject: [PATCH 33/33] fix to spotless Signed-off-by: Asif Bashar --- .../CalcitePPLToNumberFunctionTest.java | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java index 448cddeefec..bd4f2c293b6 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLToNumberFunctionTest.java @@ -49,26 +49,25 @@ public void testNumberHex() { verifyPPLToSparkSQL(root, expectedSparkSql); } + @Test + public void testNumberHexMinLimit() { + String ppl = + "source=EMP | eval long_value = tonumber('-7FFFFFFFFFFFFFFF',16) | fields long_value|head" + + " 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalSort(fetch=[1])\n" + + " LogicalProject(long_value=[TONUMBER('-7FFFFFFFFFFFFFFF':VARCHAR, 16)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "long_value=-9.223372036854776E18\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `TONUMBER`('-7FFFFFFFFFFFFFFF', 16) `long_value`\nFROM `scott`.`EMP`\nLIMIT 1"; - @Test - public void testNumberHexMinLimit() { - String ppl = - "source=EMP | eval long_value = tonumber('-7FFFFFFFFFFFFFFF',16) | fields long_value|head" - + " 1"; - RelNode root = getRelNode(ppl); - String expectedLogical = - "LogicalSort(fetch=[1])\n" - + " LogicalProject(long_value=[TONUMBER('-7FFFFFFFFFFFFFFF':VARCHAR, 16)])\n" - + " LogicalTableScan(table=[[scott, EMP]])\n"; - verifyLogical(root, expectedLogical); - String expectedResult = "long_value=-9.223372036854776E18\n"; - verifyResult(root, expectedResult); - - String expectedSparkSql = - "SELECT `TONUMBER`('-7FFFFFFFFFFFFFFF', 16) `long_value`\nFROM `scott`.`EMP`\nLIMIT 1"; - - verifyPPLToSparkSQL(root, expectedSparkSql); - } + verifyPPLToSparkSQL(root, expectedSparkSql); + } @Test public void testNumberHexMaxLimit() {