Skip to content

Commit 948ccc0

Browse files
anloweeanlowee
committed
feat: Add metadata filtering based on archive-level timestamp.
1 parent 6143799 commit 948ccc0

20 files changed

+898
-160
lines changed

‎presto-clp/pom.xml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,16 @@
6565
<scope>provided</scope>
6666
</dependency>
6767

68+
<dependency>
69+
<groupId>com.fasterxml.jackson.core</groupId>
70+
<artifactId>jackson-core</artifactId>
71+
</dependency>
72+
73+
<dependency>
74+
<groupId>com.fasterxml.jackson.core</groupId>
75+
<artifactId>jackson-databind</artifactId>
76+
</dependency>
77+
6878
<dependency>
6979
<groupId>com.facebook.presto</groupId>
7080
<artifactId>presto-spi</artifactId>

‎presto-clp/src/main/java/com/facebook/presto/plugin/clp/ClpConfig.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ public class ClpConfig
2929
private String metadataDbName;
3030
private String metadataDbUser;
3131
private String metadataDbPassword;
32+
private String metadataFilterConfig;
3233
private String metadataTablePrefix;
3334
private long metadataRefreshInterval = 60;
3435
private long metadataExpireInterval = 600;
@@ -107,6 +108,18 @@ public ClpConfig setMetadataDbPassword(String metadataDbPassword)
107108
return this;
108109
}
109110

111+
public String getMetadataFilterConfig()
112+
{
113+
return metadataFilterConfig;
114+
}
115+
116+
@Config("clp.metadata-filter-config")
117+
public ClpConfig setMetadataFilterConfig(String metadataFilterConfig)
118+
{
119+
this.metadataFilterConfig = metadataFilterConfig;
120+
return this;
121+
}
122+
110123
public String getMetadataTablePrefix()
111124
{
112125
return metadataTablePrefix;

‎presto-clp/src/main/java/com/facebook/presto/plugin/clp/ClpConnectorFactory.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,12 @@
3333
public class ClpConnectorFactory
3434
implements ConnectorFactory
3535
{
36+
public static final String CONNECTOR_NAME = "clp";
37+
3638
@Override
3739
public String getName()
3840
{
39-
return "clp";
41+
return CONNECTOR_NAME;
4042
}
4143

4244
@Override

‎presto-clp/src/main/java/com/facebook/presto/plugin/clp/ClpErrorCode.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import com.facebook.presto.spi.ErrorCodeSupplier;
1919

2020
import static com.facebook.presto.common.ErrorType.EXTERNAL;
21+
import static com.facebook.presto.common.ErrorType.USER_ERROR;
2122

2223
public enum ClpErrorCode
2324
implements ErrorCodeSupplier
@@ -26,7 +27,10 @@ public enum ClpErrorCode
2627
CLP_UNSUPPORTED_METADATA_SOURCE(1, EXTERNAL),
2728
CLP_UNSUPPORTED_SPLIT_SOURCE(2, EXTERNAL),
2829
CLP_UNSUPPORTED_TYPE(3, EXTERNAL),
29-
CLP_UNSUPPORTED_CONFIG_OPTION(4, EXTERNAL);
30+
CLP_UNSUPPORTED_CONFIG_OPTION(4, EXTERNAL),
31+
32+
CLP_METADATA_FILTER_CONFIG_NOT_FOUND(10, USER_ERROR),
33+
CLP_MANDATORY_METADATA_FILTER_NOT_VALID(11, USER_ERROR);
3034

3135
private final ErrorCode errorCode;
3236

‎presto-clp/src/main/java/com/facebook/presto/plugin/clp/ClpExpression.java‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,15 @@ public class ClpExpression
2727
{
2828
// Optional KQL query or column name representing the fully or partially translatable part of the expression.
2929
private final Optional<String> pushDownExpression;
30-
30+
// Optinal SQL string extracted from the definition, which is only made of given metadata columns.
31+
private final Optional<String> metadataSql;
3132
// The remaining (non-translatable) portion of the RowExpression, if any.
3233
private final Optional<RowExpression> remainingExpression;
3334

34-
public ClpExpression(String pushDownExpression, RowExpression remainingExpression)
35+
public ClpExpression(String pushDownExpression, String metadataSql, RowExpression remainingExpression)
3536
{
3637
this.pushDownExpression = Optional.ofNullable(pushDownExpression);
38+
this.metadataSql = Optional.ofNullable(metadataSql);
3739
this.remainingExpression = Optional.ofNullable(remainingExpression);
3840
}
3941

@@ -42,7 +44,7 @@ public ClpExpression(String pushDownExpression, RowExpression remainingExpressio
4244
*/
4345
public ClpExpression()
4446
{
45-
this(null, null);
47+
this(null, null, null);
4648
}
4749

4850
/**
@@ -52,7 +54,18 @@ public ClpExpression()
5254
*/
5355
public ClpExpression(String pushDownExpression)
5456
{
55-
this(pushDownExpression, null);
57+
this(pushDownExpression, null, null);
58+
}
59+
60+
/**
61+
* Creates a ClpExpression from a fully translatable KQL string and give it metadata SQL.
62+
*
63+
* @param pushDownExpression
64+
* @param metadataSql
65+
*/
66+
public ClpExpression(String pushDownExpression, String metadataSql)
67+
{
68+
this(pushDownExpression, metadataSql, null);
5669
}
5770

5871
/**
@@ -62,14 +75,19 @@ public ClpExpression(String pushDownExpression)
6275
*/
6376
public ClpExpression(RowExpression remainingExpression)
6477
{
65-
this(null, remainingExpression);
78+
this(null, null, remainingExpression);
6679
}
6780

6881
public Optional<String> getPushDownExpression()
6982
{
7083
return pushDownExpression;
7184
}
7285

86+
public Optional<String> getMetadataSql()
87+
{
88+
return metadataSql;
89+
}
90+
7391
public Optional<RowExpression> getRemainingExpression()
7492
{
7593
return remainingExpression;

‎presto-clp/src/main/java/com/facebook/presto/plugin/clp/ClpFilterToKqlConverter.java‎

Lines changed: 88 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import static com.facebook.presto.common.function.OperatorType.flip;
5050
import static com.facebook.presto.common.type.BooleanType.BOOLEAN;
5151
import static com.facebook.presto.plugin.clp.ClpErrorCode.CLP_PUSHDOWN_UNSUPPORTED_EXPRESSION;
52+
import static com.facebook.presto.plugin.clp.ClpUtils.KqlUtils.escapeKqlSpecialCharsForStringValue;
5253
import static com.facebook.presto.spi.relation.SpecialFormExpression.Form.AND;
5354
import static java.lang.Integer.parseInt;
5455
import static java.lang.String.format;
@@ -84,15 +85,18 @@ public class ClpFilterToKqlConverter
8485
private final StandardFunctionResolution standardFunctionResolution;
8586
private final FunctionMetadataManager functionMetadataManager;
8687
private final Map<VariableReferenceExpression, ColumnHandle> assignments;
88+
private final Set<String> metadataFilterColumns;
8789

8890
public ClpFilterToKqlConverter(
8991
StandardFunctionResolution standardFunctionResolution,
9092
FunctionMetadataManager functionMetadataManager,
91-
Map<VariableReferenceExpression, ColumnHandle> assignments)
93+
Map<VariableReferenceExpression, ColumnHandle> assignments,
94+
Set<String> metadataFilterColumns)
9295
{
9396
this.standardFunctionResolution = requireNonNull(standardFunctionResolution, "standardFunctionResolution is null");
9497
this.functionMetadataManager = requireNonNull(functionMetadataManager, "function metadata manager is null");
9598
this.assignments = requireNonNull(assignments, "assignments is null");
99+
this.metadataFilterColumns = requireNonNull(metadataFilterColumns, "metadataFilterColumns is null");
96100
}
97101

98102
@Override
@@ -114,11 +118,39 @@ public ClpExpression visitCall(CallExpression node, Void context)
114118
if (operatorType.isComparisonOperator() && operatorType != IS_DISTINCT_FROM) {
115119
return handleLogicalBinary(operatorType, node);
116120
}
121+
else if (OperatorType.BETWEEN == operatorType) {
122+
return handleBetween(operatorType, node);
123+
}
117124
}
118125

119126
return new ClpExpression(node);
120127
}
121128

129+
private ClpExpression handleBetween(OperatorType operator, CallExpression node)
130+
{
131+
if (node.getArguments().size() != 3) {
132+
throw new PrestoException(CLP_PUSHDOWN_UNSUPPORTED_EXPRESSION,
133+
"Logical binary operator must have exactly two arguments. Received: " + node);
134+
}
135+
Optional<String> variableReferenceDefinition = node.getArguments().get(0).accept(this, null).getPushDownExpression();
136+
Optional<String> lowerBoundConstantDefinition = node.getArguments().get(1).accept(this, null).getPushDownExpression();
137+
Optional<String> upperBoundConstantDefinition = node.getArguments().get(2).accept(this, null).getPushDownExpression();
138+
if (!variableReferenceDefinition.isPresent() || !lowerBoundConstantDefinition.isPresent() || !upperBoundConstantDefinition.isPresent()) {
139+
return new ClpExpression(node);
140+
}
141+
String metadataSql = null;
142+
String kql = String.format(
143+
"\"%s\" >= %s AND \"%s\" <= %s",
144+
variableReferenceDefinition.get(),
145+
lowerBoundConstantDefinition.get(),
146+
variableReferenceDefinition.get(),
147+
upperBoundConstantDefinition.get());
148+
if (metadataFilterColumns.contains(variableReferenceDefinition.get())) {
149+
metadataSql = kql;
150+
}
151+
return new ClpExpression(kql, metadataSql);
152+
}
153+
122154
@Override
123155
public ClpExpression visitConstant(ConstantExpression node, Void context)
124156
{
@@ -203,7 +235,12 @@ private ClpExpression handleNot(CallExpression node)
203235
if (expression.getRemainingExpression().isPresent() || !expression.getPushDownExpression().isPresent()) {
204236
return new ClpExpression(node);
205237
}
206-
return new ClpExpression("NOT " + expression.getPushDownExpression().get());
238+
if (expression.getMetadataSql().isPresent()) {
239+
return new ClpExpression("NOT " + expression.getPushDownExpression().get(), "NOT " + expression.getMetadataSql());
240+
}
241+
else {
242+
return new ClpExpression("NOT " + expression.getPushDownExpression().get());
243+
}
207244
}
208245

209246
/**
@@ -350,24 +387,40 @@ private ClpExpression buildClpExpression(
350387
Type literalType,
351388
RowExpression originalNode)
352389
{
390+
String metadataSql = null;
353391
if (operator.equals(EQUAL)) {
354392
if (literalType instanceof VarcharType) {
355-
return new ClpExpression(format("%s: \"%s\"", variableName, literalString));
393+
if (metadataFilterColumns.contains(variableName)) {
394+
metadataSql = format("\"%s\" = '%s'", variableName, literalString);
395+
}
396+
return new ClpExpression(format("%s: \"%s\"", variableName, escapeKqlSpecialCharsForStringValue(literalString)), metadataSql);
356397
}
357398
else {
358-
return new ClpExpression(format("%s: %s", variableName, literalString));
399+
if (metadataFilterColumns.contains(variableName)) {
400+
metadataSql = format("\"%s\" = %s", variableName, literalString);
401+
}
402+
return new ClpExpression(format("%s: %s", variableName, literalString), metadataSql);
359403
}
360404
}
361405
else if (operator.equals(NOT_EQUAL)) {
362406
if (literalType instanceof VarcharType) {
363-
return new ClpExpression(format("NOT %s: \"%s\"", variableName, literalString));
407+
if (metadataFilterColumns.contains(variableName)) {
408+
metadataSql = format("NOT \"%s\" = '%s'", variableName, literalString);
409+
}
410+
return new ClpExpression(format("NOT %s: \"%s\"", variableName, escapeKqlSpecialCharsForStringValue(literalString)), metadataSql);
364411
}
365412
else {
366-
return new ClpExpression(format("NOT %s: %s", variableName, literalString));
413+
if (metadataFilterColumns.contains(variableName)) {
414+
metadataSql = format("NOT \"%s\" = %s", variableName, literalString);
415+
}
416+
return new ClpExpression(format("NOT %s: %s", variableName, literalString), metadataSql);
367417
}
368418
}
369419
else if (LOGICAL_BINARY_OPS_FILTER.contains(operator) && !(literalType instanceof VarcharType)) {
370-
return new ClpExpression(format("%s %s %s", variableName, operator.getOperator(), literalString));
420+
if (metadataFilterColumns.contains(variableName)) {
421+
metadataSql = format("\"%s\" %s %s", variableName, operator.getOperator(), literalString);
422+
}
423+
return new ClpExpression(format("%s %s %s", variableName, operator.getOperator(), literalString), metadataSql);
371424
}
372425
return new ClpExpression(originalNode);
373426
}
@@ -568,16 +621,24 @@ private Optional<Integer> parseLengthLiteral(RowExpression lengthExpression, Str
568621
*/
569622
private ClpExpression handleAnd(SpecialFormExpression node)
570623
{
624+
StringBuilder metadataQueryBuilder = new StringBuilder();
625+
metadataQueryBuilder.append("(");
571626
StringBuilder queryBuilder = new StringBuilder();
572627
queryBuilder.append("(");
573628
List<RowExpression> remainingExpressions = new ArrayList<>();
629+
boolean hasMetadataSql = false;
574630
boolean hasPushDownExpression = false;
575631
for (RowExpression argument : node.getArguments()) {
576632
ClpExpression expression = argument.accept(this, null);
577633
if (expression.getPushDownExpression().isPresent()) {
578634
hasPushDownExpression = true;
579635
queryBuilder.append(expression.getPushDownExpression().get());
580636
queryBuilder.append(" AND ");
637+
if (expression.getMetadataSql().isPresent()) {
638+
hasMetadataSql = true;
639+
metadataQueryBuilder.append(expression.getMetadataSql().get());
640+
metadataQueryBuilder.append(" AND ");
641+
}
581642
}
582643
if (expression.getRemainingExpression().isPresent()) {
583644
remainingExpressions.add(expression.getRemainingExpression().get());
@@ -588,16 +649,21 @@ private ClpExpression handleAnd(SpecialFormExpression node)
588649
}
589650
else if (!remainingExpressions.isEmpty()) {
590651
if (remainingExpressions.size() == 1) {
591-
return new ClpExpression(queryBuilder.substring(0, queryBuilder.length() - 5) + ")", remainingExpressions.get(0));
652+
return new ClpExpression(
653+
queryBuilder.substring(0, queryBuilder.length() - 5) + ")",
654+
hasMetadataSql ? metadataQueryBuilder.substring(0, metadataQueryBuilder.length() - 5) + ")" : null,
655+
remainingExpressions.get(0));
592656
}
593657
else {
594658
return new ClpExpression(
595659
queryBuilder.substring(0, queryBuilder.length() - 5) + ")",
660+
hasMetadataSql ? metadataQueryBuilder.substring(0, metadataQueryBuilder.length() - 5) + ")" : null,
596661
new SpecialFormExpression(node.getSourceLocation(), AND, BOOLEAN, remainingExpressions));
597662
}
598663
}
599664
// Remove the last " AND " from the query
600-
return new ClpExpression(queryBuilder.substring(0, queryBuilder.length() - 5) + ")");
665+
return new ClpExpression(queryBuilder.substring(0, queryBuilder.length() - 5) + ")",
666+
hasMetadataSql ? metadataQueryBuilder.substring(0, metadataQueryBuilder.length() - 5) + ")" : null);
601667
}
602668

603669
/**
@@ -614,18 +680,30 @@ else if (!remainingExpressions.isEmpty()) {
614680
*/
615681
private ClpExpression handleOr(SpecialFormExpression node)
616682
{
683+
StringBuilder metadataQueryBuilder = new StringBuilder();
684+
metadataQueryBuilder.append("(");
617685
StringBuilder queryBuilder = new StringBuilder();
618686
queryBuilder.append("(");
687+
boolean hasAllMetadataSql = true;
619688
for (RowExpression argument : node.getArguments()) {
620689
ClpExpression expression = argument.accept(this, null);
621690
if (expression.getRemainingExpression().isPresent() || !expression.getPushDownExpression().isPresent()) {
622691
return new ClpExpression(node);
623692
}
624693
queryBuilder.append(expression.getPushDownExpression().get());
625694
queryBuilder.append(" OR ");
695+
if (hasAllMetadataSql && expression.getMetadataSql().isPresent()) {
696+
metadataQueryBuilder.append(expression.getMetadataSql().get());
697+
metadataQueryBuilder.append(" OR ");
698+
}
699+
else {
700+
hasAllMetadataSql = false;
701+
}
626702
}
627703
// Remove the last " OR " from the query
628-
return new ClpExpression(queryBuilder.substring(0, queryBuilder.length() - 4) + ")");
704+
return new ClpExpression(
705+
queryBuilder.substring(0, queryBuilder.length() - 4) + ")",
706+
hasAllMetadataSql ? metadataQueryBuilder.substring(0, metadataQueryBuilder.length() - 4) + ")" : null);
629707
}
630708

631709
/**

‎presto-clp/src/main/java/com/facebook/presto/plugin/clp/ClpMetadata.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ public ConnectorTableLayoutResult getTableLayoutForConstraint(
119119
Optional<Set<ColumnHandle>> desiredColumns)
120120
{
121121
ClpTableHandle tableHandle = (ClpTableHandle) table;
122-
ConnectorTableLayout layout = new ConnectorTableLayout(new ClpTableLayoutHandle(tableHandle, Optional.empty()));
122+
ConnectorTableLayout layout = new ConnectorTableLayout(new ClpTableLayoutHandle(tableHandle, Optional.empty(), Optional.empty()));
123123
return new ConnectorTableLayoutResult(layout, constraint.getSummary());
124124
}
125125

0 commit comments

Comments
 (0)