Skip to content

Commit a16501f

Browse files
committed
fix: Normalize timestamp literals to nanoseconds in filter pushdown
Ensure all timestamp literals are converted to nanoseconds before generating KQL and SQL filters for pushdown, regardless of their original precision (milliseconds, microseconds, etc.). Problem: Presto supports multiple timestamp precisions (TIMESTAMP with millisecond precision, TIMESTAMP_MICROSECONDS, etc.), but CLP's query engine expects timestamps in a consistent nanosecond format. Previously, timestamp literals were passed through without precision normalization, causing incorrect results when different timestamp types were used in queries. Solution: Add timestamp normalization logic in ClpFilterToKqlConverter: - tryEnsureNanosecondTimestamp(): Detects TIMESTAMP and TIMESTAMP_MICROSECONDS types - ensureNanosecondTimestamp(): Converts timestamp values to nanoseconds using: 1. Extract epoch seconds via TimestampType.getEpochSecond() 2. Extract nanosecond fraction via TimestampType.getNanos() 3. Convert to total nanoseconds: SECONDS.toNanos(seconds) + nanosFraction Applied to: - BETWEEN operator: Both lower and upper bounds - Comparison operators: All timestamp literal values (=, !=, <, >, <=, >=) This ensures consistent timestamp representation in both KQL filters (sent to CLP engine) and SQL filters (used for metadata filtering), regardless of the timestamp precision used in the original Presto query. Example: Query: WHERE timestamp BETWEEN TIMESTAMP '2025-01-01' AND TIMESTAMP '2025-01-02' Before: Literal values passed as milliseconds or microseconds After: Both bounds normalized to nanosecond representation
1 parent 95ed6c5 commit a16501f

File tree

1 file changed

+27
-2
lines changed

1 file changed

+27
-2
lines changed

presto-clp/src/main/java/com/facebook/presto/plugin/clp/optimization/ClpFilterToKqlConverter.java

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import com.facebook.presto.common.function.OperatorType;
1717
import com.facebook.presto.common.type.DecimalType;
1818
import com.facebook.presto.common.type.RowType;
19+
import com.facebook.presto.common.type.TimestampType;
1920
import com.facebook.presto.common.type.Type;
2021
import com.facebook.presto.common.type.VarcharType;
2122
import com.facebook.presto.plugin.clp.ClpColumnHandle;
@@ -64,6 +65,7 @@
6465
import static java.lang.Integer.parseInt;
6566
import static java.lang.String.format;
6667
import static java.util.Objects.requireNonNull;
68+
import static java.util.concurrent.TimeUnit.SECONDS;
6769

6870
/**
6971
* A translator to translate Presto {@link RowExpression}s into:
@@ -259,8 +261,10 @@ private ClpExpression handleBetween(CallExpression node)
259261
}
260262

261263
String variable = variableOpt.get();
262-
String lowerBound = getLiteralString((ConstantExpression) second);
263-
String upperBound = getLiteralString((ConstantExpression) third);
264+
Type lowerBoundType = second.getType();
265+
String lowerBound = tryEnsureNanosecondTimestamp(lowerBoundType, getLiteralString((ConstantExpression) second));
266+
Type upperBoundType = third.getType();
267+
String upperBound = tryEnsureNanosecondTimestamp(upperBoundType, getLiteralString((ConstantExpression) third));
264268
String kql = String.format("%s >= %s AND %s <= %s", variable, lowerBound, variable, upperBound);
265269
String metadataSqlQuery = metadataFilterColumns.contains(variable)
266270
? String.format("\"%s\" >= %s AND \"%s\" <= %s", variable, lowerBound, variable, upperBound)
@@ -443,6 +447,7 @@ private ClpExpression buildClpExpression(
443447
RowExpression originalNode)
444448
{
445449
String metadataSqlQuery = null;
450+
literalString = tryEnsureNanosecondTimestamp(literalType, literalString);
446451
if (operator.equals(EQUAL)) {
447452
if (literalType instanceof VarcharType) {
448453
return new ClpExpression(
@@ -939,6 +944,26 @@ public static boolean isClpCompatibleNumericType(Type type)
939944
|| type instanceof DecimalType;
940945
}
941946

947+
private static String tryEnsureNanosecondTimestamp(Type type, String literalString)
948+
{
949+
if (type == TIMESTAMP) {
950+
return ensureNanosecondTimestamp(TIMESTAMP, literalString);
951+
}
952+
else if (type == TIMESTAMP_MICROSECONDS) {
953+
return ensureNanosecondTimestamp(TIMESTAMP_MICROSECONDS, literalString);
954+
}
955+
return literalString;
956+
}
957+
958+
private static String ensureNanosecondTimestamp(TimestampType type, String literalString)
959+
{
960+
long literalNumber = Long.parseLong(literalString);
961+
long seconds = type.getEpochSecond(literalNumber);
962+
long nanosecondFraction = type.getNanos(literalNumber);
963+
long nanoseconds = SECONDS.toNanos(seconds) + nanosecondFraction;
964+
return Long.toString(nanoseconds);
965+
}
966+
942967
private static class SubstrInfo
943968
{
944969
String variableName;

0 commit comments

Comments
 (0)