diff --git a/presto-clp/src/main/java/com/facebook/presto/plugin/clp/ClpExpression.java b/presto-clp/src/main/java/com/facebook/presto/plugin/clp/ClpExpression.java new file mode 100644 index 0000000000000..fd74933c709fe --- /dev/null +++ b/presto-clp/src/main/java/com/facebook/presto/plugin/clp/ClpExpression.java @@ -0,0 +1,77 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.clp; + +import com.facebook.presto.spi.relation.RowExpression; + +import java.util.Optional; + +/** + * Represents the result of converting a Presto RowExpression into a CLP-compatible KQL query. In + * every case, `pushDownExpression` represents the part of the RowExpression that could be + * converted to a KQL expression, and `remainingExpression` represents the part that could not be + * converted. + */ +public class ClpExpression +{ + // Optional KQL query or column name representing the fully or partially translatable part of the expression. + private final Optional pushDownExpression; + + // The remaining (non-translatable) portion of the RowExpression, if any. + private final Optional remainingExpression; + + public ClpExpression(String pushDownExpression, RowExpression remainingExpression) + { + this.pushDownExpression = Optional.ofNullable(pushDownExpression); + this.remainingExpression = Optional.ofNullable(remainingExpression); + } + + /** + * Creates an empty ClpExpression with neither pushdown nor remaining expressions. + */ + public ClpExpression() + { + this(null, null); + } + + /** + * Creates a ClpExpression from a fully translatable KQL query or column name. + * + * @param pushDownExpression + */ + public ClpExpression(String pushDownExpression) + { + this(pushDownExpression, null); + } + + /** + * Creates a ClpExpression from a non-translatable RowExpression. + * + * @param remainingExpression + */ + public ClpExpression(RowExpression remainingExpression) + { + this(null, remainingExpression); + } + + public Optional getPushDownExpression() + { + return pushDownExpression; + } + + public Optional getRemainingExpression() + { + return remainingExpression; + } +} diff --git a/presto-clp/src/main/java/com/facebook/presto/plugin/clp/ClpFilterToKqlConverter.java b/presto-clp/src/main/java/com/facebook/presto/plugin/clp/ClpFilterToKqlConverter.java new file mode 100644 index 0000000000000..02fe7c5dc7f31 --- /dev/null +++ b/presto-clp/src/main/java/com/facebook/presto/plugin/clp/ClpFilterToKqlConverter.java @@ -0,0 +1,769 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.clp; + +import com.facebook.presto.common.function.OperatorType; +import com.facebook.presto.common.type.RowType; +import com.facebook.presto.common.type.Type; +import com.facebook.presto.common.type.VarcharType; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.PrestoException; +import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.function.FunctionMetadata; +import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.relation.CallExpression; +import com.facebook.presto.spi.relation.ConstantExpression; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.RowExpressionVisitor; +import com.facebook.presto.spi.relation.SpecialFormExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.google.common.collect.ImmutableSet; +import io.airlift.slice.Slice; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static com.facebook.presto.common.function.OperatorType.EQUAL; +import static com.facebook.presto.common.function.OperatorType.GREATER_THAN; +import static com.facebook.presto.common.function.OperatorType.GREATER_THAN_OR_EQUAL; +import static com.facebook.presto.common.function.OperatorType.IS_DISTINCT_FROM; +import static com.facebook.presto.common.function.OperatorType.LESS_THAN; +import static com.facebook.presto.common.function.OperatorType.LESS_THAN_OR_EQUAL; +import static com.facebook.presto.common.function.OperatorType.NEGATION; +import static com.facebook.presto.common.function.OperatorType.NOT_EQUAL; +import static com.facebook.presto.common.function.OperatorType.flip; +import static com.facebook.presto.common.type.BooleanType.BOOLEAN; +import static com.facebook.presto.plugin.clp.ClpErrorCode.CLP_PUSHDOWN_UNSUPPORTED_EXPRESSION; +import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.AND; +import static java.lang.Integer.parseInt; +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +/** + * A translator to translate Presto RowExpressions into KQL (Kibana Query Language) filters used as + * CLP queries. This is used primarily for pushing down supported filters to the CLP engine. This + * class implements the RowExpressionVisitor interface and recursively walks Presto filter + * expressions, attempting to convert supported expressions into corresponding KQL filter strings. + * Any part of the expression that cannot be translated is preserved as a "remaining expression" for + * potential fallback processing. + *

+ * Supported translations include: + *
    + *
  • Comparisons between variables and constants (e.g., =, !=, <, >, <=, >=).
    • + *
  • String pattern matches using LIKE with constant patterns only. Patterns that begin and + * end with % (i.e., "^%[^%_]*%$") are not supported.
    • + *
  • Membership checks using IN with a list of constants only.
    • + *
  • NULL checks via IS NULL.
    • + *
  • Substring comparisons (e.g., SUBSTR(x, start, len) = "val") against a + * constant.
    • + *
  • Dereferencing fields from row-typed variables.
    • + *
  • Logical operators AND, OR, and NOT.
    • + *
+ */ +public class ClpFilterToKqlConverter + implements RowExpressionVisitor +{ + private static final Set LOGICAL_BINARY_OPS_FILTER = + ImmutableSet.of(EQUAL, NOT_EQUAL, LESS_THAN, LESS_THAN_OR_EQUAL, GREATER_THAN, GREATER_THAN_OR_EQUAL); + + private final StandardFunctionResolution standardFunctionResolution; + private final FunctionMetadataManager functionMetadataManager; + private final Map assignments; + + public ClpFilterToKqlConverter( + StandardFunctionResolution standardFunctionResolution, + FunctionMetadataManager functionMetadataManager, + Map assignments) + { + this.standardFunctionResolution = requireNonNull(standardFunctionResolution, "standardFunctionResolution is null"); + this.functionMetadataManager = requireNonNull(functionMetadataManager, "function metadata manager is null"); + this.assignments = requireNonNull(assignments, "assignments is null"); + } + + @Override + public ClpExpression visitCall(CallExpression node, Void context) + { + FunctionHandle functionHandle = node.getFunctionHandle(); + if (standardFunctionResolution.isNotFunction(functionHandle)) { + return handleNot(node); + } + + if (standardFunctionResolution.isLikeFunction(functionHandle)) { + return handleLike(node); + } + + FunctionMetadata functionMetadata = functionMetadataManager.getFunctionMetadata(node.getFunctionHandle()); + Optional operatorTypeOptional = functionMetadata.getOperatorType(); + if (operatorTypeOptional.isPresent()) { + OperatorType operatorType = operatorTypeOptional.get(); + if (operatorType.isComparisonOperator() && operatorType != IS_DISTINCT_FROM) { + return handleLogicalBinary(operatorType, node); + } + } + + return new ClpExpression(node); + } + + @Override + public ClpExpression visitConstant(ConstantExpression node, Void context) + { + return new ClpExpression(getLiteralString(node)); + } + + @Override + public ClpExpression visitVariableReference(VariableReferenceExpression node, Void context) + { + return new ClpExpression(getVariableName(node)); + } + + @Override + public ClpExpression visitSpecialForm(SpecialFormExpression node, Void context) + { + switch (node.getForm()) { + case AND: + return handleAnd(node); + case OR: + return handleOr(node); + case IN: + return handleIn(node); + case IS_NULL: + return handleIsNull(node); + case DEREFERENCE: + return handleDereference(node); + default: + return new ClpExpression(node); + } + } + + @Override + public ClpExpression visitExpression(RowExpression node, Void context) + { + // For all other expressions, return the original expression + return new ClpExpression(node); + } + + /** + * Extracts the string representation of a constant expression. + * + * @param literal the constant expression + * @return the string representation of the literal + */ + private String getLiteralString(ConstantExpression literal) + { + if (literal.getValue() instanceof Slice) { + return ((Slice) literal.getValue()).toStringUtf8(); + } + return literal.toString(); + } + + /** + * Retrieves the original column name from a variable reference. + * + * @param variable the variable reference expression + * @return the original column name as a string + */ + private String getVariableName(VariableReferenceExpression variable) + { + return ((ClpColumnHandle) assignments.get(variable)).getOriginalColumnName(); + } + + /** + * Handles the logical NOT expression. + *

+ * Example: NOT (col1 = 5) → NOT col1: 5 + * + * @param node the NOT call expression + * @return a ClpExpression containing either the equivalent KQL query, or the original + * expression if it couldn't be translated + */ + private ClpExpression handleNot(CallExpression node) + { + if (node.getArguments().size() != 1) { + throw new PrestoException(CLP_PUSHDOWN_UNSUPPORTED_EXPRESSION, + "NOT operator must have exactly one argument. Received: " + node); + } + + RowExpression input = node.getArguments().get(0); + ClpExpression expression = input.accept(this, null); + if (expression.getRemainingExpression().isPresent() || !expression.getPushDownExpression().isPresent()) { + return new ClpExpression(node); + } + return new ClpExpression("NOT " + expression.getPushDownExpression().get()); + } + + /** + * Handles LIKE expressions. + *

+ * Converts SQL LIKE patterns into equivalent KQL queries using * (for + * %) and ? (for _). Only supports constant or casted + * constant patterns. + *

+ * Example: col1 LIKE 'a_bc%' → col1: "a?bc*" + * + * @param node the LIKE call expression + * @return a ClpExpression containing either the equivalent KQL query, or the original + * expression if it couldn't be translated + */ + private ClpExpression handleLike(CallExpression node) + { + if (node.getArguments().size() != 2) { + throw new PrestoException(CLP_PUSHDOWN_UNSUPPORTED_EXPRESSION, "LIKE operator must have exactly two arguments. Received: " + node); + } + ClpExpression variable = node.getArguments().get(0).accept(this, null); + if (!variable.getPushDownExpression().isPresent()) { + return new ClpExpression(node); + } + + String variableName = variable.getPushDownExpression().get(); + RowExpression argument = node.getArguments().get(1); + + String pattern; + if (argument instanceof ConstantExpression) { + ConstantExpression literal = (ConstantExpression) argument; + pattern = getLiteralString(literal); + } + else if (argument instanceof CallExpression) { + CallExpression callExpression = (CallExpression) argument; + if (!standardFunctionResolution.isCastFunction(callExpression.getFunctionHandle())) { + return new ClpExpression(node); + } + if (callExpression.getArguments().size() != 1) { + throw new PrestoException(CLP_PUSHDOWN_UNSUPPORTED_EXPRESSION, "CAST function must have exactly one argument. Received: " + callExpression); + } + if (!(callExpression.getArguments().get(0) instanceof ConstantExpression)) { + return new ClpExpression(node); + } + pattern = getLiteralString((ConstantExpression) callExpression.getArguments().get(0)); + } + else { + return new ClpExpression(node); + } + pattern = pattern.replace("%", "*").replace("_", "?"); + return new ClpExpression(format("%s: \"%s\"", variableName, pattern)); + } + + /** + * Handles logical binary operators (e.g., =, !=, <, >) between two expressions. + *

+ * Supports constant values on either side and flips the operator if necessary. Also delegates + * to a substring handler for SUBSTR(x, ...) = 'value' patterns. + * + * @param operator the binary operator (e.g., EQUAL, NOT_EQUAL) + * @param node the call expression representing the binary operation + * @return a ClpExpression containing either the equivalent KQL query, or the original + * expression if it couldn't be translated + */ + private ClpExpression handleLogicalBinary(OperatorType operator, CallExpression node) + { + if (node.getArguments().size() != 2) { + throw new PrestoException(CLP_PUSHDOWN_UNSUPPORTED_EXPRESSION, + "Logical binary operator must have exactly two arguments. Received: " + node); + } + RowExpression left = node.getArguments().get(0); + RowExpression right = node.getArguments().get(1); + + ClpExpression maybeLeftSubstring = tryInterpretSubstringEquality(operator, left, right); + if (maybeLeftSubstring.getPushDownExpression().isPresent()) { + return maybeLeftSubstring; + } + + ClpExpression maybeRightSubstring = tryInterpretSubstringEquality(operator, right, left); + if (maybeRightSubstring.getPushDownExpression().isPresent()) { + return maybeRightSubstring; + } + + ClpExpression leftExpression = left.accept(this, null); + ClpExpression rightExpression = right.accept(this, null); + Optional leftDefinition = leftExpression.getPushDownExpression(); + Optional rightDefinition = rightExpression.getPushDownExpression(); + if (!leftDefinition.isPresent() || !rightDefinition.isPresent()) { + return new ClpExpression(node); + } + + boolean leftIsConstant = (left instanceof ConstantExpression); + boolean rightIsConstant = (right instanceof ConstantExpression); + + Type leftType = left.getType(); + Type rightType = right.getType(); + + if (rightIsConstant) { + return buildClpExpression( + leftDefinition.get(), // variable + rightDefinition.get(), // literal + operator, + rightType, + node); + } + else if (leftIsConstant) { + OperatorType newOperator = flip(operator); + return buildClpExpression( + rightDefinition.get(), // variable + leftDefinition.get(), // literal + newOperator, + leftType, + node); + } + // fallback + return new ClpExpression(node); + } + + /** + * Builds a CLP expression from a basic comparison between a variable and a constant. + *

+ * Handles different operator types and formats them appropriately based on whether the literal + * is a string or a non-string type. + *

+ * Examples: + *
    + *
  • col = 'abc' → col: "abc"
    • + *
  • col != 42 → NOT col: 42
    • + *
  • 5 < col → col > 5
    • + *
+ * + * @param variableName name of the variable + * @param literalString string representation of the literal + * @param operator the comparison operator + * @param literalType the type of the literal + * @param originalNode the original RowExpression node + * @return a ClpExpression containing either the equivalent KQL query, or the original + * expression if it couldn't be translated + */ + private ClpExpression buildClpExpression( + String variableName, + String literalString, + OperatorType operator, + Type literalType, + RowExpression originalNode) + { + if (operator.equals(EQUAL)) { + if (literalType instanceof VarcharType) { + return new ClpExpression(format("%s: \"%s\"", variableName, literalString)); + } + else { + return new ClpExpression(format("%s: %s", variableName, literalString)); + } + } + else if (operator.equals(NOT_EQUAL)) { + if (literalType instanceof VarcharType) { + return new ClpExpression(format("NOT %s: \"%s\"", variableName, literalString)); + } + else { + return new ClpExpression(format("NOT %s: %s", variableName, literalString)); + } + } + else if (LOGICAL_BINARY_OPS_FILTER.contains(operator) && !(literalType instanceof VarcharType)) { + return new ClpExpression(format("%s %s %s", variableName, operator.getOperator(), literalString)); + } + return new ClpExpression(originalNode); + } + + /** + * Checks whether the given expression matches the pattern + * SUBSTR(x, ...) = 'someString', and if so, attempts to convert it into a KQL + * query using wildcards and constructs a CLP expression. + * + * @param operator the comparison operator (should be EQUAL) + * @param possibleSubstring the left or right expression, possibly a SUBSTR call + * @param possibleLiteral the opposite expression, possibly a string constant + * @return a ClpExpression containing either the equivalent KQL query, or nothing if it couldn't + * be translated + */ + private ClpExpression tryInterpretSubstringEquality( + OperatorType operator, + RowExpression possibleSubstring, + RowExpression possibleLiteral) + { + if (!operator.equals(EQUAL)) { + return new ClpExpression(); + } + + if (!(possibleSubstring instanceof CallExpression) || + !(possibleLiteral instanceof ConstantExpression)) { + return new ClpExpression(); + } + + Optional maybeSubstringCall = parseSubstringCall((CallExpression) possibleSubstring); + if (!maybeSubstringCall.isPresent()) { + return new ClpExpression(); + } + + String targetString = getLiteralString((ConstantExpression) possibleLiteral); + return interpretSubstringEquality(maybeSubstringCall.get(), targetString); + } + + /** + * Parses a SUBSTR(x, start [, length]) call into a SubstrInfo object if valid. + * + * @param callExpression the call expression to inspect + * @return an Optional containing SubstrInfo if the expression is a valid SUBSTR call + */ + private Optional parseSubstringCall(CallExpression callExpression) + { + FunctionMetadata functionMetadata = functionMetadataManager.getFunctionMetadata(callExpression.getFunctionHandle()); + String functionName = functionMetadata.getName().getObjectName(); + if (!functionName.equals("substr")) { + return Optional.empty(); + } + + int argCount = callExpression.getArguments().size(); + if (argCount < 2 || argCount > 3) { + return Optional.empty(); + } + + ClpExpression variable = callExpression.getArguments().get(0).accept(this, null); + if (!variable.getPushDownExpression().isPresent()) { + return Optional.empty(); + } + + String varName = variable.getPushDownExpression().get(); + RowExpression startExpression = callExpression.getArguments().get(1); + RowExpression lengthExpression = null; + if (argCount == 3) { + lengthExpression = callExpression.getArguments().get(2); + } + + return Optional.of(new SubstrInfo(varName, startExpression, lengthExpression)); + } + + /** + * Converts a SUBSTR(x, start [, length]) = 'someString' into a KQL-style wildcard + * query. + *

+ * Examples: + *
    + *
  • SUBSTR(message, 1, 3) = 'abc' → message: "abc*"
    • + *
  • SUBSTR(message, 4, 3) = 'abc' → message: "???abc*"
    • + *
  • SUBSTR(message, 2) = 'hello' → message: "?hello"
    • + *
  • SUBSTR(message, -5) = 'hello' → message: "*hello"
    • + *
+ * + * @param info parsed SUBSTR call info + * @param targetString the literal string being compared to + * @return a ClpExpression containing either the equivalent KQL query, or nothing if it couldn't + * be translated + */ + private ClpExpression interpretSubstringEquality(SubstrInfo info, String targetString) + { + if (info.lengthExpression != null) { + Optional maybeStart = parseIntValue(info.startExpression); + Optional maybeLen = parseLengthLiteral(info.lengthExpression, targetString); + + if (maybeStart.isPresent() && maybeLen.isPresent()) { + int start = maybeStart.get(); + int len = maybeLen.get(); + if (start > 0 && len == targetString.length()) { + StringBuilder result = new StringBuilder(); + result.append(info.variableName).append(": \""); + for (int i = 1; i < start; i++) { + result.append("?"); + } + result.append(targetString).append("*\""); + return new ClpExpression(result.toString()); + } + } + } + else { + Optional maybeStart = parseIntValue(info.startExpression); + if (maybeStart.isPresent()) { + int start = maybeStart.get(); + if (start > 0) { + StringBuilder result = new StringBuilder(); + result.append(info.variableName).append(": \""); + for (int i = 1; i < start; i++) { + result.append("?"); + } + result.append(targetString).append("\""); + return new ClpExpression(result.toString()); + } + if (start == -targetString.length()) { + return new ClpExpression(format("%s: \"*%s\"", info.variableName, targetString)); + } + } + } + + return new ClpExpression(); + } + + /** + * Attempts to parse a RowExpression as an integer constant. + * + * @param expression the row expression to parse + * @return an Optional containing the integer value if it could be parsed + */ + private Optional parseIntValue(RowExpression expression) + { + if (expression instanceof ConstantExpression) { + try { + return Optional.of(parseInt(getLiteralString((ConstantExpression) expression))); + } + catch (NumberFormatException ignored) { + } + } + else if (expression instanceof CallExpression) { + CallExpression call = (CallExpression) expression; + FunctionMetadata functionMetadata = functionMetadataManager.getFunctionMetadata(call.getFunctionHandle()); + Optional operatorTypeOptional = functionMetadata.getOperatorType(); + if (operatorTypeOptional.isPresent() && operatorTypeOptional.get().equals(NEGATION)) { + RowExpression arg0 = call.getArguments().get(0); + if (arg0 instanceof ConstantExpression) { + try { + return Optional.of(-parseInt(getLiteralString((ConstantExpression) arg0))); + } + catch (NumberFormatException ignored) { + } + } + } + } + return Optional.empty(); + } + + /** + * Attempts to parse the length expression and match it against the target string's length. + * + * @param lengthExpression the expression representing the length parameter + * @param targetString the target string to compare length against + * @return an Optional containing the length if it matches targetString.length() + */ + private Optional parseLengthLiteral(RowExpression lengthExpression, String targetString) + { + if (lengthExpression instanceof ConstantExpression) { + String val = getLiteralString((ConstantExpression) lengthExpression); + try { + int parsed = parseInt(val); + if (parsed == targetString.length()) { + return Optional.of(parsed); + } + } + catch (NumberFormatException ignored) { + } + } + return Optional.empty(); + } + + /** + * Handles the logical AND expression. + *

+ * Combines all definable child expressions into a single KQL query joined by AND. Any + * unsupported children are collected into the remaining expression. + *

+ * Example: col1 = 5 AND col2 = 'abc' → (col1: 5 AND col2: "abc") + * + * @param node the AND special form expression + * @return a ClpExpression containing the KQL query and any remaining sub-expressions + */ + private ClpExpression handleAnd(SpecialFormExpression node) + { + StringBuilder queryBuilder = new StringBuilder(); + queryBuilder.append("("); + List remainingExpressions = new ArrayList<>(); + boolean hasPushDownExpression = false; + for (RowExpression argument : node.getArguments()) { + ClpExpression expression = argument.accept(this, null); + if (expression.getPushDownExpression().isPresent()) { + hasPushDownExpression = true; + queryBuilder.append(expression.getPushDownExpression().get()); + queryBuilder.append(" AND "); + } + if (expression.getRemainingExpression().isPresent()) { + remainingExpressions.add(expression.getRemainingExpression().get()); + } + } + if (!hasPushDownExpression) { + return new ClpExpression(node); + } + else if (!remainingExpressions.isEmpty()) { + if (remainingExpressions.size() == 1) { + return new ClpExpression(queryBuilder.substring(0, queryBuilder.length() - 5) + ")", remainingExpressions.get(0)); + } + else { + return new ClpExpression( + queryBuilder.substring(0, queryBuilder.length() - 5) + ")", + new SpecialFormExpression(node.getSourceLocation(), AND, BOOLEAN, remainingExpressions)); + } + } + // Remove the last " AND " from the query + return new ClpExpression(queryBuilder.substring(0, queryBuilder.length() - 5) + ")"); + } + + /** + * Handles the logical OR expression. + *

+ * Combines all fully convertible child expressions into a single KQL query joined by OR. + * Falls back to the original node if any child cannot be converted. + *

+ * Example: col1 = 5 OR col1 = 10 → (col1: 5 OR col1: 10) + * + * @param node the OR special form expression + * @return a ClpExpression containing either the equivalent KQL query, or the original + * expression if it couldn't be fully translated + */ + private ClpExpression handleOr(SpecialFormExpression node) + { + StringBuilder queryBuilder = new StringBuilder(); + queryBuilder.append("("); + for (RowExpression argument : node.getArguments()) { + ClpExpression expression = argument.accept(this, null); + if (expression.getRemainingExpression().isPresent() || !expression.getPushDownExpression().isPresent()) { + return new ClpExpression(node); + } + queryBuilder.append(expression.getPushDownExpression().get()); + queryBuilder.append(" OR "); + } + // Remove the last " OR " from the query + return new ClpExpression(queryBuilder.substring(0, queryBuilder.length() - 4) + ")"); + } + + /** + * Handles the IN predicate. + *

+ * Example: col1 IN (1, 2, 3) → (col1: 1 OR col1: 2 OR col1: 3) + * + * @param node the IN special form expression + * @return a ClpExpression containing either the equivalent KQL query, or the original + * expression if it couldn't be translated + */ + private ClpExpression handleIn(SpecialFormExpression node) + { + ClpExpression variable = node.getArguments().get(0).accept(this, null); + if (!variable.getPushDownExpression().isPresent()) { + return new ClpExpression(node); + } + String variableName = variable.getPushDownExpression().get(); + StringBuilder queryBuilder = new StringBuilder(); + queryBuilder.append("("); + for (RowExpression argument : node.getArguments().subList(1, node.getArguments().size())) { + if (!(argument instanceof ConstantExpression)) { + return new ClpExpression(node); + } + ConstantExpression literal = (ConstantExpression) argument; + String literalString = getLiteralString(literal); + queryBuilder.append(variableName).append(": "); + if (literal.getType() instanceof VarcharType) { + queryBuilder.append("\"").append(literalString).append("\""); + } + else { + queryBuilder.append(literalString); + } + queryBuilder.append(" OR "); + } + + // Remove the last " OR " from the query + return new ClpExpression(queryBuilder.substring(0, queryBuilder.length() - 4) + ")"); + } + + /** + * Handles the IS NULL predicate. + *

+ * Example: col1 IS NULL → NOT col1: * + * + * @param node the IS_NULL special form expression + * @return a ClpExpression containing either the equivalent KQL query, or the original + * expression if it couldn't be translated + */ + private ClpExpression handleIsNull(SpecialFormExpression node) + { + if (node.getArguments().size() != 1) { + throw new PrestoException(CLP_PUSHDOWN_UNSUPPORTED_EXPRESSION, + "IS NULL operator must have exactly one argument. Received: " + node); + } + + ClpExpression expression = node.getArguments().get(0).accept(this, null); + if (!expression.getPushDownExpression().isPresent()) { + return new ClpExpression(node); + } + + String variableName = expression.getPushDownExpression().get(); + return new ClpExpression(format("NOT %s: *", variableName)); + } + + /** + * Handles dereference expressions on RowTypes (e.g., col.row_field). + *

+ * Converts nested row field accesses into dot-separated KQL-compatible field names. + *

+ * Example: address.city (from a RowType 'address') → address.city + * + * @param expression the dereference expression ({@link SpecialFormExpression} or + * {@link VariableReferenceExpression}) + * @return a ClpExpression containing either the dot-separated field name, or the original + * expression if it couldn't be translated + */ + private ClpExpression handleDereference(RowExpression expression) + { + if (expression instanceof VariableReferenceExpression) { + return expression.accept(this, null); + } + + if (!(expression instanceof SpecialFormExpression)) { + return new ClpExpression(expression); + } + + SpecialFormExpression specialForm = (SpecialFormExpression) expression; + List arguments = specialForm.getArguments(); + if (arguments.size() != 2) { + throw new PrestoException(CLP_PUSHDOWN_UNSUPPORTED_EXPRESSION, "DEREFERENCE expects 2 arguments"); + } + + RowExpression base = arguments.get(0); + RowExpression index = arguments.get(1); + if (!(index instanceof ConstantExpression)) { + throw new PrestoException(CLP_PUSHDOWN_UNSUPPORTED_EXPRESSION, "DEREFERENCE index must be a constant"); + } + + ConstantExpression constExpr = (ConstantExpression) index; + Object value = constExpr.getValue(); + if (!(value instanceof Long)) { + throw new PrestoException(CLP_PUSHDOWN_UNSUPPORTED_EXPRESSION, "DEREFERENCE index constant is not a long"); + } + + int fieldIndex = ((Long) value).intValue(); + + Type baseType = base.getType(); + if (!(baseType instanceof RowType)) { + throw new PrestoException(CLP_PUSHDOWN_UNSUPPORTED_EXPRESSION, "DEREFERENCE base is not a RowType: " + baseType); + } + + RowType rowType = (RowType) baseType; + if (fieldIndex < 0 || fieldIndex >= rowType.getFields().size()) { + throw new PrestoException(CLP_PUSHDOWN_UNSUPPORTED_EXPRESSION, "Invalid field index " + fieldIndex + " for RowType: " + rowType); + } + + RowType.Field field = rowType.getFields().get(fieldIndex); + String fieldName = field.getName().orElse("field" + fieldIndex); + + ClpExpression baseString = handleDereference(base); + if (!baseString.getPushDownExpression().isPresent()) { + return new ClpExpression(expression); + } + return new ClpExpression(baseString.getPushDownExpression().get() + "." + fieldName); + } + + private static class SubstrInfo + { + String variableName; + RowExpression startExpression; + RowExpression lengthExpression; + + SubstrInfo(String variableName, RowExpression start, RowExpression length) + { + this.variableName = variableName; + this.startExpression = start; + this.lengthExpression = length; + } + } +} diff --git a/presto-clp/src/main/java/com/facebook/presto/plugin/clp/ClpPlanOptimizer.java b/presto-clp/src/main/java/com/facebook/presto/plugin/clp/ClpPlanOptimizer.java new file mode 100644 index 0000000000000..adab0bf71c9a8 --- /dev/null +++ b/presto-clp/src/main/java/com/facebook/presto/plugin/clp/ClpPlanOptimizer.java @@ -0,0 +1,111 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.clp; + +import com.facebook.airlift.log.Logger; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorPlanOptimizer; +import com.facebook.presto.spi.ConnectorPlanRewriter; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.VariableAllocator; +import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.plan.FilterNode; +import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; + +import java.util.Map; +import java.util.Optional; + +import static com.facebook.presto.spi.ConnectorPlanRewriter.rewriteWith; +import static java.util.Objects.requireNonNull; + +public class ClpPlanOptimizer + implements ConnectorPlanOptimizer +{ + private static final Logger log = Logger.get(ClpPlanOptimizer.class); + private final FunctionMetadataManager functionManager; + private final StandardFunctionResolution functionResolution; + + public ClpPlanOptimizer(FunctionMetadataManager functionManager, StandardFunctionResolution functionResolution) + { + this.functionManager = requireNonNull(functionManager, "functionManager is null"); + this.functionResolution = requireNonNull(functionResolution, "functionResolution is null"); + } + + @Override + public PlanNode optimize(PlanNode maxSubplan, ConnectorSession session, VariableAllocator variableAllocator, PlanNodeIdAllocator idAllocator) + { + return rewriteWith(new Rewriter(idAllocator), maxSubplan); + } + + private class Rewriter + extends ConnectorPlanRewriter + { + private final PlanNodeIdAllocator idAllocator; + + public Rewriter(PlanNodeIdAllocator idAllocator) + { + this.idAllocator = idAllocator; + } + + @Override + public PlanNode visitFilter(FilterNode node, RewriteContext context) + { + if (!(node.getSource() instanceof TableScanNode)) { + return node; + } + + TableScanNode tableScanNode = (TableScanNode) node.getSource(); + Map assignments = tableScanNode.getAssignments(); + TableHandle tableHandle = tableScanNode.getTable(); + ClpTableHandle clpTableHandle = (ClpTableHandle) tableHandle.getConnectorHandle(); + ClpExpression clpExpression = node.getPredicate() + .accept(new ClpFilterToKqlConverter(functionResolution, functionManager, assignments), null); + Optional kqlQuery = clpExpression.getPushDownExpression(); + Optional remainingPredicate = clpExpression.getRemainingExpression(); + if (!kqlQuery.isPresent()) { + return node; + } + log.debug("KQL query: %s", kqlQuery.get()); + ClpTableLayoutHandle clpTableLayoutHandle = new ClpTableLayoutHandle(clpTableHandle, kqlQuery); + TableScanNode newTableScanNode = new TableScanNode( + tableScanNode.getSourceLocation(), + idAllocator.getNextId(), + new TableHandle( + tableHandle.getConnectorId(), + clpTableHandle, + tableHandle.getTransaction(), + Optional.of(clpTableLayoutHandle)), + tableScanNode.getOutputVariables(), + tableScanNode.getAssignments(), + tableScanNode.getTableConstraints(), + tableScanNode.getCurrentConstraint(), + tableScanNode.getEnforcedConstraint(), + tableScanNode.getCteMaterializationInfo()); + if (!remainingPredicate.isPresent()) { + return newTableScanNode; + } + + return new FilterNode(node.getSourceLocation(), + idAllocator.getNextId(), + newTableScanNode, + remainingPredicate.get()); + } + } +} diff --git a/presto-clp/src/main/java/com/facebook/presto/plugin/clp/ClpPlanOptimizerProvider.java b/presto-clp/src/main/java/com/facebook/presto/plugin/clp/ClpPlanOptimizerProvider.java new file mode 100644 index 0000000000000..f6f166eb7f657 --- /dev/null +++ b/presto-clp/src/main/java/com/facebook/presto/plugin/clp/ClpPlanOptimizerProvider.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.clp; + +import com.facebook.presto.spi.ConnectorPlanOptimizer; +import com.facebook.presto.spi.connector.ConnectorPlanOptimizerProvider; +import com.facebook.presto.spi.function.FunctionMetadataManager; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.google.common.collect.ImmutableSet; + +import javax.inject.Inject; + +import java.util.Set; + +public class ClpPlanOptimizerProvider + implements ConnectorPlanOptimizerProvider +{ + private final FunctionMetadataManager functionManager; + private final StandardFunctionResolution functionResolution; + + @Inject + public ClpPlanOptimizerProvider(FunctionMetadataManager functionManager, StandardFunctionResolution functionResolution) + { + this.functionManager = functionManager; + this.functionResolution = functionResolution; + } + + @Override + public Set getLogicalPlanOptimizers() + { + return ImmutableSet.of(); + } + + @Override + public Set getPhysicalPlanOptimizers() + { + return ImmutableSet.of(new ClpPlanOptimizer(functionManager, functionResolution)); + } +} diff --git a/presto-clp/src/main/java/com/facebook/presto/plugin/clp/split/ClpMySqlSplitProvider.java b/presto-clp/src/main/java/com/facebook/presto/plugin/clp/split/ClpMySqlSplitProvider.java index ac646a061ba4b..964a992230e36 100644 --- a/presto-clp/src/main/java/com/facebook/presto/plugin/clp/split/ClpMySqlSplitProvider.java +++ b/presto-clp/src/main/java/com/facebook/presto/plugin/clp/split/ClpMySqlSplitProvider.java @@ -38,10 +38,10 @@ public class ClpMySqlSplitProvider public static final String ARCHIVES_TABLE_COLUMN_ID = "id"; // Table suffixes - public static final String ARCHIVE_TABLE_SUFFIX = "_archives"; + public static final String ARCHIVES_TABLE_SUFFIX = "_archives"; // SQL templates - private static final String SQL_SELECT_ARCHIVES_TEMPLATE = format("SELECT `%s` FROM `%%s%%s%s`", ARCHIVES_TABLE_COLUMN_ID, ARCHIVE_TABLE_SUFFIX); + private static final String SQL_SELECT_ARCHIVES_TEMPLATE = format("SELECT `%s` FROM `%%s%%s%s`", ARCHIVES_TABLE_COLUMN_ID, ARCHIVES_TABLE_SUFFIX); private static final Logger log = Logger.get(ClpMySqlSplitProvider.class); diff --git a/presto-clp/src/test/java/com/facebook/presto/plugin/clp/ClpMetadataDbSetUp.java b/presto-clp/src/test/java/com/facebook/presto/plugin/clp/ClpMetadataDbSetUp.java index 6b7220cc1dd43..cebe4afcfa0a7 100644 --- a/presto-clp/src/test/java/com/facebook/presto/plugin/clp/ClpMetadataDbSetUp.java +++ b/presto-clp/src/test/java/com/facebook/presto/plugin/clp/ClpMetadataDbSetUp.java @@ -38,7 +38,7 @@ import static com.facebook.presto.plugin.clp.metadata.ClpMySqlMetadataProvider.DATASETS_TABLE_COLUMN_NAME; import static com.facebook.presto.plugin.clp.metadata.ClpMySqlMetadataProvider.DATASETS_TABLE_SUFFIX; import static com.facebook.presto.plugin.clp.split.ClpMySqlSplitProvider.ARCHIVES_TABLE_COLUMN_ID; -import static com.facebook.presto.plugin.clp.split.ClpMySqlSplitProvider.ARCHIVE_TABLE_SUFFIX; +import static com.facebook.presto.plugin.clp.split.ClpMySqlSplitProvider.ARCHIVES_TABLE_SUFFIX; import static java.lang.String.format; import static java.util.UUID.randomUUID; import static org.testng.Assert.fail; @@ -53,7 +53,7 @@ public final class ClpMetadataDbSetUp private static final Logger log = Logger.get(ClpMetadataDbSetUp.class); private static final String DATASETS_TABLE_NAME = METADATA_DB_TABLE_PREFIX + DATASETS_TABLE_SUFFIX; - private static final String ARCHIVE_TABLE_COLUMN_PAGINATION_ID = "pagination_id"; + private static final String ARCHIVES_TABLE_COLUMN_PAGINATION_ID = "pagination_id"; private ClpMetadataDbSetUp() { @@ -122,7 +122,7 @@ public static ClpMetadata setupMetadata(DbHandle dbHandle, Map> splits) { final String metadataDbUrl = format(METADATA_DB_URL_TEMPLATE, dbHandle.dbPath); - final String archiveTableFormat = METADATA_DB_TABLE_PREFIX + "%s" + ARCHIVE_TABLE_SUFFIX; + final String archiveTableFormat = METADATA_DB_TABLE_PREFIX + "%s" + ARCHIVES_TABLE_SUFFIX; try (Connection conn = DriverManager.getConnection(metadataDbUrl, METADATA_DB_USER, METADATA_DB_PASSWORD); Statement stmt = conn.createStatement()) { createDatasetsTable(stmt); @@ -138,7 +138,7 @@ public static ClpMySqlSplitProvider setupSplit(DbHandle dbHandle, Map 0", "fare > 0", null, sessionHolder); + testFilter("fare >= 0", "fare >= 0", null, sessionHolder); + testFilter("fare < 0", "fare < 0", null, sessionHolder); + testFilter("fare <= 0", "fare <= 0", null, sessionHolder); + testFilter("fare = 0", "fare: 0", null, sessionHolder); + testFilter("fare != 0", "NOT fare: 0", null, sessionHolder); + testFilter("fare <> 0", "NOT fare: 0", null, sessionHolder); + testFilter("0 < fare", "fare > 0", null, sessionHolder); + testFilter("0 <= fare", "fare >= 0", null, sessionHolder); + testFilter("0 > fare", "fare < 0", null, sessionHolder); + testFilter("0 >= fare", "fare <= 0", null, sessionHolder); + testFilter("0 = fare", "fare: 0", null, sessionHolder); + testFilter("0 != fare", "NOT fare: 0", null, sessionHolder); + testFilter("0 <> fare", "NOT fare: 0", null, sessionHolder); + } + + @Test + public void testOrPushdown() + { + SessionHolder sessionHolder = new SessionHolder(); + + testFilter("fare > 0 OR city.Name like 'b%'", "(fare > 0 OR city.Name: \"b*\")", null, sessionHolder); + testFilter( + "lower(city.Region.Name) = 'hello world' OR city.Region.Id != 1", + null, + "(lower(city.Region.Name) = 'hello world' OR city.Region.Id != 1)", + sessionHolder); + + // Multiple ORs + testFilter( + "fare > 0 OR city.Name like 'b%' OR lower(city.Region.Name) = 'hello world' OR city.Region.Id != 1", + null, + "fare > 0 OR city.Name like 'b%' OR lower(city.Region.Name) = 'hello world' OR city.Region.Id != 1", + sessionHolder); + testFilter( + "fare > 0 OR city.Name like 'b%' OR city.Region.Id != 1", + "((fare > 0 OR city.Name: \"b*\") OR NOT city.Region.Id: 1)", + null, + sessionHolder); + } + + @Test + public void testAndPushdown() + { + SessionHolder sessionHolder = new SessionHolder(); + + testFilter("fare > 0 AND city.Name like 'b%'", "(fare > 0 AND city.Name: \"b*\")", null, sessionHolder); + testFilter( + "lower(city.Region.Name) = 'hello world' AND city.Region.Id != 1", + "(NOT city.Region.Id: 1)", + "lower(city.Region.Name) = 'hello world'", + sessionHolder); + + // Multiple ANDs + testFilter( + "fare > 0 AND city.Name like 'b%' AND lower(city.Region.Name) = 'hello world' AND city.Region.Id != 1", + "(((fare > 0 AND city.Name: \"b*\")) AND NOT city.Region.Id: 1)", + "(lower(city.Region.Name) = 'hello world')", + sessionHolder); + testFilter( + "fare > 0 AND city.Name like '%b%' AND lower(city.Region.Name) = 'hello world' AND city.Region.Id != 1", + "(((fare > 0)) AND NOT city.Region.Id: 1)", + "city.Name like '%b%' AND lower(city.Region.Name) = 'hello world'", + sessionHolder); + } + + @Test + public void testNotPushdown() + { + SessionHolder sessionHolder = new SessionHolder(); + + testFilter("city.Region.Name NOT LIKE 'hello%'", "NOT city.Region.Name: \"hello*\"", null, sessionHolder); + testFilter("NOT (city.Region.Name LIKE 'hello%')", "NOT city.Region.Name: \"hello*\"", null, sessionHolder); + testFilter("city.Name != 'hello world'", "NOT city.Name: \"hello world\"", null, sessionHolder); + testFilter("city.Name <> 'hello world'", "NOT city.Name: \"hello world\"", null, sessionHolder); + testFilter("NOT (city.Name = 'hello world')", "NOT city.Name: \"hello world\"", null, sessionHolder); + testFilter("fare != 0", "NOT fare: 0", null, sessionHolder); + testFilter("fare <> 0", "NOT fare: 0", null, sessionHolder); + testFilter("NOT (fare = 0)", "NOT fare: 0", null, sessionHolder); + + // Multiple NOTs + testFilter("NOT (NOT fare = 0)", "NOT NOT fare: 0", null, sessionHolder); + testFilter("NOT (fare = 0 AND city.Name = 'hello world')", "NOT (fare: 0 AND city.Name: \"hello world\")", null, sessionHolder); + testFilter("NOT (fare = 0 OR city.Name = 'hello world')", "NOT (fare: 0 OR city.Name: \"hello world\")", null, sessionHolder); + } + + @Test + public void testInPushdown() + { + SessionHolder sessionHolder = new SessionHolder(); + + testFilter("city.Name IN ('hello world', 'hello world 2')", "(city.Name: \"hello world\" OR city.Name: \"hello world 2\")", null, sessionHolder); + } + + @Test + public void testIsNullPushdown() + { + SessionHolder sessionHolder = new SessionHolder(); + + testFilter("city.Name IS NULL", "NOT city.Name: *", null, sessionHolder); + testFilter("city.Name IS NOT NULL", "NOT NOT city.Name: *", null, sessionHolder); + testFilter("NOT (city.Name IS NULL)", "NOT NOT city.Name: *", null, sessionHolder); + } + + @Test + public void testComplexPushdown() + { + SessionHolder sessionHolder = new SessionHolder(); + + testFilter( + "(fare > 0 OR city.Name like 'b%') AND (lower(city.Region.Name) = 'hello world' OR city.Name IS NULL)", + "((fare > 0 OR city.Name: \"b*\"))", + "(lower(city.Region.Name) = 'hello world' OR city.Name IS NULL)", + sessionHolder); + testFilter( + "city.Region.Id = 1 AND (fare > 0 OR city.Name NOT like 'b%') AND (lower(city.Region.Name) = 'hello world' OR city.Name IS NULL)", + "((city.Region.Id: 1 AND (fare > 0 OR NOT city.Name: \"b*\")))", + "lower(city.Region.Name) = 'hello world' OR city.Name IS NULL", + sessionHolder); + } + + private void testFilter(String sqlExpression, String expectedKqlExpression, String expectedRemainingExpression, SessionHolder sessionHolder) + { + RowExpression actualExpression = getRowExpression(sqlExpression, sessionHolder); + ClpExpression clpExpression = actualExpression.accept(new ClpFilterToKqlConverter(standardFunctionResolution, functionAndTypeManager, variableToColumnHandleMap), null); + Optional kqlExpression = clpExpression.getPushDownExpression(); + Optional remainingExpression = clpExpression.getRemainingExpression(); + if (expectedKqlExpression != null) { + assertTrue(kqlExpression.isPresent()); + assertEquals(kqlExpression.get(), expectedKqlExpression); + } + + if (expectedRemainingExpression != null) { + assertTrue(remainingExpression.isPresent()); + assertEquals(remainingExpression.get(), getRowExpression(expectedRemainingExpression, sessionHolder)); + } + else { + assertFalse(remainingExpression.isPresent()); + } + } +} diff --git a/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpQueryBase.java b/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpQueryBase.java new file mode 100644 index 0000000000000..1b759ab3e1282 --- /dev/null +++ b/presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpQueryBase.java @@ -0,0 +1,146 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.plugin.clp; + +import com.facebook.presto.Session; +import com.facebook.presto.SystemSessionProperties; +import com.facebook.presto.common.block.BlockEncodingManager; +import com.facebook.presto.common.type.RowType; +import com.facebook.presto.common.type.Type; +import com.facebook.presto.metadata.AnalyzePropertyManager; +import com.facebook.presto.metadata.CatalogManager; +import com.facebook.presto.metadata.ColumnPropertyManager; +import com.facebook.presto.metadata.FunctionAndTypeManager; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.metadata.MetadataManager; +import com.facebook.presto.metadata.SchemaPropertyManager; +import com.facebook.presto.metadata.TablePropertyManager; +import com.facebook.presto.spi.ColumnHandle; +import com.facebook.presto.spi.ConnectorSession; +import com.facebook.presto.spi.SchemaTableName; +import com.facebook.presto.spi.function.StandardFunctionResolution; +import com.facebook.presto.spi.relation.RowExpression; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.facebook.presto.sql.parser.ParsingOptions; +import com.facebook.presto.sql.parser.SqlParser; +import com.facebook.presto.sql.planner.TypeProvider; +import com.facebook.presto.sql.relational.FunctionResolution; +import com.facebook.presto.sql.relational.SqlToRowExpressionTranslator; +import com.facebook.presto.sql.tree.Expression; +import com.facebook.presto.sql.tree.NodeRef; +import com.facebook.presto.testing.TestingSession; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; + +import static com.facebook.presto.common.type.BigintType.BIGINT; +import static com.facebook.presto.common.type.BooleanType.BOOLEAN; +import static com.facebook.presto.common.type.DoubleType.DOUBLE; +import static com.facebook.presto.common.type.VarcharType.VARCHAR; +import static com.facebook.presto.metadata.FunctionAndTypeManager.createTestFunctionAndTypeManager; +import static com.facebook.presto.metadata.SessionPropertyManager.createTestingSessionPropertyManager; +import static com.facebook.presto.spi.WarningCollector.NOOP; +import static com.facebook.presto.sql.ExpressionUtils.rewriteIdentifiersToSymbolReferences; +import static com.facebook.presto.sql.analyzer.ExpressionAnalyzer.getExpressionTypes; +import static com.facebook.presto.sql.parser.ParsingOptions.DecimalLiteralTreatment.AS_DECIMAL; +import static com.facebook.presto.testing.TestingConnectorSession.SESSION; +import static com.facebook.presto.transaction.InMemoryTransactionManager.createTestTransactionManager; +import static java.util.stream.Collectors.toMap; + +public class TestClpQueryBase +{ + protected static final FunctionAndTypeManager functionAndTypeManager = createTestFunctionAndTypeManager(); + protected static final StandardFunctionResolution standardFunctionResolution = new FunctionResolution(functionAndTypeManager.getFunctionAndTypeResolver()); + protected static final Metadata metadata = new MetadataManager( + functionAndTypeManager, + new BlockEncodingManager(), + createTestingSessionPropertyManager(), + new SchemaPropertyManager(), + new TablePropertyManager(), + new ColumnPropertyManager(), + new AnalyzePropertyManager(), + createTestTransactionManager(new CatalogManager())); + + protected static final ClpTableHandle table = new ClpTableHandle(new SchemaTableName("default", "test"), "", ClpTableHandle.StorageType.FS); + protected static final ClpColumnHandle city = new ClpColumnHandle( + "city", + RowType.from(ImmutableList.of( + RowType.field("Region", RowType.from(ImmutableList.of( + RowType.field("Id", BIGINT), + RowType.field("Name", VARCHAR)))), + RowType.field("Name", VARCHAR))), + true); + protected static final ClpColumnHandle fare = new ClpColumnHandle("fare", DOUBLE, true); + protected static final ClpColumnHandle isHoliday = new ClpColumnHandle("isHoliday", BOOLEAN, true); + protected static final Map variableToColumnHandleMap = + Stream.of(city, fare, isHoliday) + .collect(toMap( + ch -> new VariableReferenceExpression(Optional.empty(), ch.getColumnName(), ch.getColumnType()), + ch -> ch)); + protected final TypeProvider typeProvider = TypeProvider.fromVariables(variableToColumnHandleMap.keySet()); + + public static Expression expression(String sql) + { + return rewriteIdentifiersToSymbolReferences( + new SqlParser().createExpression(sql, ParsingOptions.builder().setDecimalLiteralTreatment(AS_DECIMAL).build())); + } + + protected RowExpression toRowExpression(Expression expression, TypeProvider typeProvider, Session session) + { + Map, Type> expressionTypes = getExpressionTypes( + session, + metadata, + new SqlParser(), + typeProvider, + expression, + ImmutableMap.of(), + NOOP); + return SqlToRowExpressionTranslator.translate(expression, expressionTypes, ImmutableMap.of(), functionAndTypeManager, session); + } + + protected RowExpression getRowExpression(String sqlExpression, SessionHolder sessionHolder) + { + return toRowExpression(expression(sqlExpression), typeProvider, sessionHolder.getSession()); + } + + protected RowExpression getRowExpression(String sqlExpression, TypeProvider typeProvider, SessionHolder sessionHolder) + { + return toRowExpression(expression(sqlExpression), typeProvider, sessionHolder.getSession()); + } + + protected static class SessionHolder + { + private final ConnectorSession connectorSession; + private final Session session; + + public SessionHolder() + { + connectorSession = SESSION; + session = TestingSession.testSessionBuilder(createTestingSessionPropertyManager(new SystemSessionProperties().getSessionProperties())).build(); + } + + public ConnectorSession getConnectorSession() + { + return connectorSession; + } + + public Session getSession() + { + return session; + } + } +}