Skip to content

Commit ba3674a

Browse files
anloweeanlowee
committed
Finish
1 parent 58f885e commit ba3674a

File tree

3 files changed

+184
-374
lines changed

3 files changed

+184
-374
lines changed

‎presto-clp/src/test/java/com/facebook/presto/plugin/clp/ClpQueryRunner.java‎

Lines changed: 109 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,19 @@
1818
import com.facebook.presto.tests.DistributedQueryRunner;
1919
import com.google.common.collect.ImmutableMap;
2020

21+
import java.io.File;
22+
import java.io.FileOutputStream;
2123
import java.net.URI;
2224
import java.util.Map;
2325
import java.util.Optional;
26+
import java.util.UUID;
2427
import java.util.function.BiFunction;
2528

29+
import static com.facebook.presto.plugin.clp.ClpConfig.SplitFilterProviderType;
2630
import static com.facebook.presto.testing.TestingSession.testSessionBuilder;
31+
import static java.lang.String.format;
32+
import static org.testng.Assert.assertTrue;
33+
import static org.testng.Assert.fail;
2734

2835
public class ClpQueryRunner
2936
{
@@ -43,7 +50,6 @@ public static DistributedQueryRunner createQueryRunner(
4350
Optional<BiFunction<Integer, URI, Process>> externalWorkerLauncher)
4451
throws Exception
4552
{
46-
log.info("Creating CLP query runner with default session");
4753
return createQueryRunner(
4854
createDefaultSession(),
4955
metadataDbUrl,
@@ -54,6 +60,29 @@ public static DistributedQueryRunner createQueryRunner(
5460
externalWorkerLauncher);
5561
}
5662

63+
public static DistributedQueryRunner createQueryRunner(
64+
String metadataDbUrl,
65+
String metadataDbUser,
66+
String metadataDbPassword,
67+
String metadataDbTablePrefix,
68+
Optional<String> splitFilterProvider,
69+
Optional<String> splitFilterConfigPath,
70+
Optional<Integer> workerCount,
71+
Optional<BiFunction<Integer, URI, Process>> externalWorkerLauncher)
72+
throws Exception
73+
{
74+
return createQueryRunner(
75+
createDefaultSession(),
76+
metadataDbUrl,
77+
metadataDbUser,
78+
metadataDbPassword,
79+
metadataDbTablePrefix,
80+
splitFilterProvider,
81+
splitFilterConfigPath,
82+
workerCount,
83+
externalWorkerLauncher);
84+
}
85+
5786
public static DistributedQueryRunner createQueryRunner(
5887
Session session,
5988
String metadataDbUrl,
@@ -63,25 +92,97 @@ public static DistributedQueryRunner createQueryRunner(
6392
Optional<Integer> workerCount,
6493
Optional<BiFunction<Integer, URI, Process>> externalWorkerLauncher)
6594
throws Exception
95+
{
96+
return createQueryRunner(
97+
session,
98+
metadataDbUrl,
99+
metadataDbUser,
100+
metadataDbPassword,
101+
metadataDbTablePrefix,
102+
Optional.empty(),
103+
Optional.empty(),
104+
workerCount,
105+
externalWorkerLauncher);
106+
}
107+
108+
public static DistributedQueryRunner createQueryRunner(
109+
Session session,
110+
String metadataDbUrl,
111+
String metadataDbUser,
112+
String metadataDbPassword,
113+
String metadataDbTablePrefix,
114+
Optional<String> splitFilterProvider,
115+
Optional<String> splitFilterConfigPath,
116+
Optional<Integer> workerCount,
117+
Optional<BiFunction<Integer, URI, Process>> externalWorkerLauncher)
118+
throws Exception
66119
{
67120
DistributedQueryRunner clpQueryRunner = DistributedQueryRunner.builder(session)
68-
.setNodeCount(workerCount.orElse(DEFAULT_NUM_OF_WORKERS))
69-
.setExternalWorkerLauncher(externalWorkerLauncher)
70-
.build();
71-
Map<String, String> clpProperties = ImmutableMap.<String, String>builder()
121+
.setNodeCount(workerCount.orElse(DEFAULT_NUM_OF_WORKERS))
122+
.setExternalWorkerLauncher(externalWorkerLauncher)
123+
.build();
124+
ImmutableMap.Builder<String, String> clpProperties = ImmutableMap.<String, String>builder()
72125
.put("clp.metadata-provider-type", "mysql")
73126
.put("clp.metadata-db-url", metadataDbUrl)
74127
.put("clp.metadata-db-user", metadataDbUser)
75128
.put("clp.metadata-db-password", metadataDbPassword)
76129
.put("clp.metadata-table-prefix", metadataDbTablePrefix)
77-
.put("clp.split-provider-type", "mysql")
78-
.build();
130+
.put("clp.split-provider-type", splitFilterProvider.orElse(SplitFilterProviderType.MYSQL.name()));
131+
splitFilterConfigPath.ifPresent(s -> clpProperties.put("clp.split-filter-config", s));
79132

133+
Map<String, String> clpPropertiesMap = clpProperties.build();
134+
log.info("Creating query runner with clp properties:");
135+
for (Map.Entry<String, String> entry : clpPropertiesMap.entrySet()) {
136+
log.info("\t%s=%s", entry.getKey(), entry.getValue());
137+
}
80138
clpQueryRunner.installPlugin(new ClpPlugin());
81-
clpQueryRunner.createCatalog(CLP_CATALOG, CLP_CONNECTOR, clpProperties);
139+
clpQueryRunner.createCatalog(CLP_CATALOG, CLP_CONNECTOR, clpPropertiesMap);
82140
return clpQueryRunner;
83141
}
84142

143+
/**
144+
* Create a config file with the given literal string of the configuration.
145+
*
146+
* @param configuration the given literal string of the configuration, which will be written into the config file
147+
* @return the string of the config file path
148+
*/
149+
public static String createConfigFile(String configuration)
150+
{
151+
UUID uuid = UUID.randomUUID();
152+
File file = new File(format("/tmp/config-%s", uuid));
153+
try {
154+
boolean fileCreated = file.createNewFile();
155+
assertTrue(fileCreated);
156+
}
157+
catch (Exception e) {
158+
fail(e.getMessage());
159+
}
160+
161+
if (!file.exists() || !file.canWrite()) {
162+
fail("Cannot create split filter config file");
163+
}
164+
try (FileOutputStream fos = new FileOutputStream(file)) {
165+
fos.write(configuration.getBytes());
166+
}
167+
catch (Exception e) {
168+
fail(e.getMessage());
169+
}
170+
return file.getAbsolutePath();
171+
}
172+
173+
/**
174+
* Delete a config file by the given path.
175+
*
176+
* @param configFilePath the path of the config file
177+
*/
178+
public static void deleteConfigFile(String configFilePath)
179+
{
180+
File splitFilterConfigFile = new File(configFilePath);
181+
if (splitFilterConfigFile.exists()) {
182+
assertTrue(splitFilterConfigFile.delete());
183+
}
184+
}
185+
85186
/**
86187
* Creates a default mock session for query use.
87188
*

‎presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpComputePushDown.java‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,9 @@
3939
import java.util.regex.Pattern;
4040

4141
import static com.facebook.presto.execution.QueryState.RUNNING;
42+
import static com.facebook.presto.plugin.clp.ClpQueryRunner.createConfigFile;
4243
import static com.facebook.presto.plugin.clp.ClpQueryRunner.createQueryRunner;
44+
import static com.facebook.presto.plugin.clp.ClpQueryRunner.deleteConfigFile;
4345
import static com.facebook.presto.plugin.clp.metadata.ClpSchemaTreeNodeType.Boolean;
4446
import static com.facebook.presto.plugin.clp.metadata.ClpSchemaTreeNodeType.DateString;
4547
import static com.facebook.presto.plugin.clp.metadata.ClpSchemaTreeNodeType.Float;
@@ -64,11 +66,13 @@ public class TestClpComputePushDown
6466
private DistributedQueryRunner queryRunner;
6567
private SqlQueryManager queryManager;
6668
private DispatchManager dispatchManager;
69+
private String splitFilterConfigFilePath;
6770

6871
@BeforeClass
6972
public void setUp()
7073
throws Exception
7174
{
75+
// Set up metadata database
7276
mockMetadataDatabase = ClpMockMetadataDatabase
7377
.builder()
7478
.build();
@@ -92,11 +96,29 @@ public void setUp()
9296
Float,
9397
Boolean,
9498
DateString))));
99+
// Set up split filter config
100+
String splitFilterConfigJsonString = "{\n" +
101+
" \"clp\": [\n" +
102+
" {\n" +
103+
" \"columnName\": \"fare\",\n" +
104+
" \"customOptions\": {\n" +
105+
" \"rangeMapping\": {\n" +
106+
" \"lowerBound\": \"fare_lb\",\n" +
107+
" \"upperBound\": \"fare_ub\"\n" +
108+
" }\n" +
109+
" },\n" +
110+
" \"required\": false\n" +
111+
" }\n" +
112+
" ]\n" +
113+
"}";
114+
splitFilterConfigFilePath = createConfigFile(splitFilterConfigJsonString);
95115
queryRunner = createQueryRunner(
96116
mockMetadataDatabase.getUrl(),
97117
mockMetadataDatabase.getUsername(),
98118
mockMetadataDatabase.getPassword(),
99119
mockMetadataDatabase.getTablePrefix(),
120+
Optional.empty(),
121+
Optional.of(splitFilterConfigFilePath),
100122
Optional.of(0),
101123
Optional.empty());
102124
queryManager = (SqlQueryManager) queryRunner.getCoordinator().getQueryManager();
@@ -117,6 +139,7 @@ public void tearDown()
117139
if (null != mockMetadataDatabase) {
118140
mockMetadataDatabase.teardown();
119141
}
142+
deleteConfigFile(splitFilterConfigFilePath);
120143
}
121144

122145
@Test
@@ -284,6 +307,43 @@ public void testComplexPushDown()
284307
"lower(city.Region.Name) = 'hello world' OR city.Name IS NULL");
285308
}
286309

310+
@Test
311+
public void testSplitFilterPushDown()
312+
{
313+
// Normal case
314+
testPushDown(
315+
"(fare > 0 AND city.Name like 'b%')",
316+
"(fare > 0.0 AND city.Name: \"b*\")",
317+
null,
318+
"(fare_ub > 0.0)");
319+
320+
// With BETWEEN
321+
testPushDown(
322+
"((fare BETWEEN 0 AND 5) AND city.Name like 'b%')",
323+
"(fare >= 0.0 AND fare <= 5.0 AND city.Name: \"b*\")",
324+
null,
325+
"(fare_ub >= 0.0 AND fare_lb <= 5.0)");
326+
327+
// The cases of that the metadata filter column exist but cannot be push down
328+
testPushDown(
329+
"(fare > 0 OR city.Name like 'b%')",
330+
"(fare > 0.0 OR city.Name: \"b*\")",
331+
null,
332+
null);
333+
testPushDown(
334+
"(fare > 0 AND city.Name like 'b%') OR city.Region.Id = 1",
335+
"((city.Region.id: 1 OR fare > 0.0) AND (city.Region.id: 1 OR city.Name: \"b*\"))",
336+
null,
337+
null);
338+
339+
// Complicated case
340+
testPushDown(
341+
"fare = 0 AND (city.Name like 'b%' OR city.Region.Id = 1)",
342+
"(fare: 0.0 AND (city.Name: \"b*\" OR city.Region.id: 1))",
343+
null,
344+
"((fare_lb <= 0.0 AND fare_ub >= 0.0))");
345+
}
346+
287347
@Test
288348
public void testClpWildcardUdf()
289349
{
@@ -313,12 +373,23 @@ public void testClpWildcardUdf()
313373
}
314374

315375
private void testPushDown(String filter, String expectedPushDown, String expectedRemaining)
376+
{
377+
testPushDown(filter, expectedPushDown, expectedRemaining, true, null);
378+
}
379+
380+
private void testPushDown(String filter, String expectedPushDown, String expectedRemaining, String expectedSplitFilterPushDown)
381+
{
382+
testPushDown(filter, expectedPushDown, expectedRemaining, false, expectedSplitFilterPushDown);
383+
}
384+
385+
private void testPushDown(String filter, String expectedPushDown, String expectedRemaining, boolean ignoreSplitFilterPushDown, String expectedSplitFilterPushDown)
316386
{
317387
try {
318388
// We first execute a query using the original filter and look for the FilterNode (for remaining expression)
319389
// and TableScanNode (for KQL pushdown and split filter pushdown)
320390
QueryId originalQueryId = createAndPlanQuery(filter);
321391
String actualPushDown = null;
392+
String actualSplitFilterPushDown = null;
322393
RowExpression actualRemainingExpression = null;
323394
Plan originalQueryPlan = queryManager.getQueryPlan(originalQueryId);
324395
for (Map.Entry<PlanNodeId, PlanNode> entry : originalQueryPlan.getPlanIdNodeMap().entrySet()) {
@@ -330,9 +401,13 @@ private void testPushDown(String filter, String expectedPushDown, String expecte
330401
clpTableLayoutHandle = tryGetClpTableLayoutHandleFromTableScanNode(entry.getValue());
331402
if (clpTableLayoutHandle != null && actualPushDown == null) {
332403
actualPushDown = clpTableLayoutHandle.getKqlQuery().orElse(null);
404+
actualSplitFilterPushDown = clpTableLayoutHandle.getMetadataSql().orElse(null);
333405
}
334406
}
335407
assertEquals(actualPushDown, expectedPushDown);
408+
if (!ignoreSplitFilterPushDown) {
409+
assertEquals(actualSplitFilterPushDown, expectedSplitFilterPushDown);
410+
}
336411
if (expectedRemaining != null) {
337412
assertNotNull(actualRemainingExpression);
338413
// Since the remaining expression cannot be simply compared by given String, we have to first convert

0 commit comments

Comments
 (0)