Skip to content

Commit 1c38e95

Browse files
anloweeanlowee
authored
Merge branch 'release-0.293-clp-connector' into xwei/handle-special-chars
2 parents c17c680 + 02fe345 commit 1c38e95

File tree

4 files changed

+107
-64
lines changed

4 files changed

+107
-64
lines changed

‎presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpMetadata.java‎

Lines changed: 41 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,16 @@
1515

1616
import com.facebook.presto.common.type.ArrayType;
1717
import com.facebook.presto.common.type.RowType;
18+
import com.facebook.presto.plugin.clp.metadata.ClpMetadataProvider;
19+
import com.facebook.presto.plugin.clp.metadata.ClpMySqlMetadataProvider;
20+
import com.facebook.presto.plugin.clp.mockdb.ClpMockMetadataDatabase;
21+
import com.facebook.presto.plugin.clp.mockdb.table.ColumnMetadataTableRows;
1822
import com.facebook.presto.spi.ColumnMetadata;
1923
import com.facebook.presto.spi.ConnectorTableMetadata;
2024
import com.facebook.presto.spi.SchemaTableName;
2125
import com.google.common.collect.ImmutableList;
2226
import com.google.common.collect.ImmutableMap;
2327
import com.google.common.collect.ImmutableSet;
24-
import org.apache.commons.math3.util.Pair;
2528
import org.testng.annotations.AfterMethod;
2629
import org.testng.annotations.BeforeMethod;
2730
import org.testng.annotations.Test;
@@ -34,9 +37,6 @@
3437
import static com.facebook.presto.common.type.DoubleType.DOUBLE;
3538
import static com.facebook.presto.common.type.VarcharType.VARCHAR;
3639
import static com.facebook.presto.plugin.clp.ClpMetadata.DEFAULT_SCHEMA_NAME;
37-
import static com.facebook.presto.plugin.clp.ClpMetadataDbSetUp.DbHandle;
38-
import static com.facebook.presto.plugin.clp.ClpMetadataDbSetUp.getDbHandle;
39-
import static com.facebook.presto.plugin.clp.ClpMetadataDbSetUp.setupMetadata;
4040
import static com.facebook.presto.plugin.clp.metadata.ClpSchemaTreeNodeType.Boolean;
4141
import static com.facebook.presto.plugin.clp.metadata.ClpSchemaTreeNodeType.ClpString;
4242
import static com.facebook.presto.plugin.clp.metadata.ClpSchemaTreeNodeType.Float;
@@ -49,32 +49,50 @@
4949
@Test(singleThreaded = true)
5050
public class TestClpMetadata
5151
{
52-
private static final String TABLE_NAME = "test";
53-
private DbHandle dbHandle;
52+
private static final String TABLE_NAME = "test_metadata";
53+
private ClpMockMetadataDatabase mockMetadataDatabase;
5454
private ClpMetadata metadata;
5555

5656
@BeforeMethod
5757
public void setUp()
5858
{
59-
dbHandle = getDbHandle("metadata_testdb");
60-
metadata = setupMetadata(
61-
dbHandle,
62-
ImmutableMap.of(
63-
TABLE_NAME,
64-
ImmutableList.of(
65-
new Pair<>("a", Integer),
66-
new Pair<>("a", ClpString),
67-
new Pair<>("b", Float),
68-
new Pair<>("b", ClpString),
69-
new Pair<>("c.d", Boolean),
70-
new Pair<>("c.e", VarString),
71-
new Pair<>("f.g.h", UnstructuredArray))));
59+
mockMetadataDatabase = ClpMockMetadataDatabase
60+
.builder()
61+
.build();
62+
mockMetadataDatabase.addTableToDatasetsTableIfNotExist(ImmutableList.of(TABLE_NAME));
63+
mockMetadataDatabase.addColumnMetadata(ImmutableMap.of(TABLE_NAME, new ColumnMetadataTableRows(
64+
ImmutableList.of(
65+
"a",
66+
"a",
67+
"b",
68+
"b",
69+
"c.d",
70+
"c.e",
71+
"f.g.h"),
72+
ImmutableList.of(
73+
Integer,
74+
ClpString,
75+
Float,
76+
ClpString,
77+
Boolean,
78+
VarString,
79+
UnstructuredArray))));
80+
ClpConfig config = new ClpConfig()
81+
.setPolymorphicTypeEnabled(true)
82+
.setMetadataDbUrl(mockMetadataDatabase.getUrl())
83+
.setMetadataDbUser(mockMetadataDatabase.getUsername())
84+
.setMetadataDbPassword(mockMetadataDatabase.getPassword())
85+
.setMetadataTablePrefix(mockMetadataDatabase.getTablePrefix());
86+
ClpMetadataProvider metadataProvider = new ClpMySqlMetadataProvider(config);
87+
metadata = new ClpMetadata(config, metadataProvider);
7288
}
7389

7490
@AfterMethod
7591
public void tearDown()
7692
{
77-
ClpMetadataDbSetUp.tearDown(dbHandle);
93+
if (null != mockMetadataDatabase) {
94+
mockMetadataDatabase.teardown();
95+
}
7896
}
7997

8098
@Test
@@ -86,9 +104,9 @@ public void testListSchemaNames()
86104
@Test
87105
public void testListTables()
88106
{
89-
HashSet<SchemaTableName> tables = new HashSet<>();
90-
tables.add(new SchemaTableName(DEFAULT_SCHEMA_NAME, TABLE_NAME));
91-
assertEquals(new HashSet<>(metadata.listTables(SESSION, Optional.empty())), tables);
107+
ImmutableSet.Builder<SchemaTableName> builder = ImmutableSet.builder();
108+
builder.add(new SchemaTableName(DEFAULT_SCHEMA_NAME, TABLE_NAME));
109+
assertEquals(new HashSet<>(metadata.listTables(SESSION, Optional.empty())), builder.build());
92110
}
93111

94112
@Test

‎presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpSplit.java‎

Lines changed: 50 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -13,75 +13,88 @@
1313
*/
1414
package com.facebook.presto.plugin.clp;
1515

16+
import com.facebook.presto.plugin.clp.mockdb.ClpMockMetadataDatabase;
17+
import com.facebook.presto.plugin.clp.mockdb.table.ArchivesTableRows;
18+
import com.facebook.presto.plugin.clp.split.ClpMySqlSplitProvider;
1619
import com.facebook.presto.plugin.clp.split.ClpSplitProvider;
1720
import com.facebook.presto.spi.SchemaTableName;
1821
import com.google.common.collect.ImmutableList;
19-
import com.google.common.collect.ImmutableSet;
22+
import com.google.common.collect.ImmutableMap;
2023
import org.testng.annotations.AfterMethod;
2124
import org.testng.annotations.BeforeMethod;
2225
import org.testng.annotations.Test;
2326

24-
import java.util.ArrayList;
25-
import java.util.HashMap;
2627
import java.util.List;
2728
import java.util.Map;
2829
import java.util.Optional;
2930

3031
import static com.facebook.presto.plugin.clp.ClpMetadata.DEFAULT_SCHEMA_NAME;
3132
import static com.facebook.presto.plugin.clp.ClpMetadataDbSetUp.ARCHIVES_STORAGE_DIRECTORY_BASE;
32-
import static com.facebook.presto.plugin.clp.ClpMetadataDbSetUp.ArchivesTableRow;
33-
import static com.facebook.presto.plugin.clp.ClpMetadataDbSetUp.DbHandle;
34-
import static com.facebook.presto.plugin.clp.ClpMetadataDbSetUp.getDbHandle;
35-
import static com.facebook.presto.plugin.clp.ClpMetadataDbSetUp.setupSplit;
3633
import static com.google.common.collect.ImmutableList.toImmutableList;
37-
import static com.google.common.collect.ImmutableSet.toImmutableSet;
34+
import static java.lang.String.format;
3835
import static org.testng.Assert.assertEquals;
3936

4037
@Test(singleThreaded = true)
4138
public class TestClpSplit
4239
{
43-
private DbHandle dbHandle;
40+
private ClpMockMetadataDatabase mockMetadataDatabase;
4441
private ClpSplitProvider clpSplitProvider;
45-
private Map<String, List<ArchivesTableRow>> tableSplits;
42+
private Map<String, ArchivesTableRows> tableSplits;
4643

4744
@BeforeMethod
4845
public void setUp()
4946
{
50-
dbHandle = getDbHandle("split_testdb");
51-
tableSplits = new HashMap<>();
52-
53-
int numKeys = 3;
54-
int numValuesPerKey = 10;
55-
56-
for (int i = 0; i < numKeys; i++) {
57-
String key = "test_" + i;
58-
List<ArchivesTableRow> values = new ArrayList<>();
59-
60-
for (int j = 0; j < numValuesPerKey; j++) {
47+
mockMetadataDatabase = ClpMockMetadataDatabase
48+
.builder()
49+
.build();
50+
ImmutableList.Builder<String> tableNamesBuilder = ImmutableList.builder();
51+
ImmutableMap.Builder<String, ArchivesTableRows> splitsMapBuilder = ImmutableMap.builder();
52+
53+
int numTables = 3;
54+
int numSplitsPerTable = 10;
55+
56+
for (int i = 0; i < numTables; i++) {
57+
String tableName = "test_split_" + i;
58+
tableNamesBuilder.add(tableName);
59+
60+
ImmutableList.Builder<String> idsBuilder = ImmutableList.builder();
61+
ImmutableList.Builder<Long> beginTimestampsBuilder = ImmutableList.builder();
62+
ImmutableList.Builder<Long> endTimestampsBuilder = ImmutableList.builder();
63+
for (int j = 0; j < numSplitsPerTable; j++) {
6164
// We generate synthetic begin_timestamp and end_timestamp values for each split
6265
// by offsetting two base timestamps (1700000000000L and 1705000000000L) with a
6366
// fixed increment per split (10^10 * j).
64-
values.add(new ArchivesTableRow(
65-
"id_" + j,
66-
1700000000000L + 10000000000L * j,
67-
1705000000000L + 10000000000L * j));
67+
idsBuilder.add(format("id_%s", j));
68+
beginTimestampsBuilder.add(1700000000000L + 10000000000L * j);
69+
endTimestampsBuilder.add(1705000000000L + 10000000000L * j);
6870
}
69-
70-
tableSplits.put(key, values);
71+
splitsMapBuilder.put(tableName, new ArchivesTableRows(idsBuilder.build(), beginTimestampsBuilder.build(), endTimestampsBuilder.build()));
7172
}
72-
clpSplitProvider = setupSplit(dbHandle, tableSplits);
73+
74+
mockMetadataDatabase.addTableToDatasetsTableIfNotExist(tableNamesBuilder.build());
75+
tableSplits = splitsMapBuilder.build();
76+
mockMetadataDatabase.addSplits(tableSplits);
77+
ClpConfig config = new ClpConfig()
78+
.setPolymorphicTypeEnabled(true)
79+
.setMetadataDbUrl(mockMetadataDatabase.getUrl())
80+
.setMetadataDbUser(mockMetadataDatabase.getUsername())
81+
.setMetadataDbPassword(mockMetadataDatabase.getPassword())
82+
.setMetadataTablePrefix(mockMetadataDatabase.getTablePrefix());
83+
clpSplitProvider = new ClpMySqlSplitProvider(config);
7384
}
7485

7586
@AfterMethod
7687
public void tearDown()
7788
{
78-
ClpMetadataDbSetUp.tearDown(dbHandle);
89+
if (null != mockMetadataDatabase) {
90+
mockMetadataDatabase.teardown();
91+
}
7992
}
8093

8194
@Test
8295
public void testListSplits()
8396
{
84-
for (Map.Entry<String, List<ArchivesTableRow>> entry : tableSplits.entrySet()) {
97+
for (Map.Entry<String, ArchivesTableRows> entry : tableSplits.entrySet()) {
8598
// Without metadata filters
8699
compareListSplitsResult(entry, Optional.empty(), ImmutableList.of(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
87100

@@ -118,29 +131,25 @@ public void testListSplits()
118131
}
119132

120133
private void compareListSplitsResult(
121-
Map.Entry<String, List<ArchivesTableRow>> entry,
134+
Map.Entry<String, ArchivesTableRows> entry,
122135
Optional<String> metadataSql,
123-
List<Integer> expectedSplitIds)
136+
List<Integer> expectedSplitIndexes)
124137
{
125138
String tableName = entry.getKey();
126139
String tablePath = ARCHIVES_STORAGE_DIRECTORY_BASE + tableName;
127140
ClpTableLayoutHandle layoutHandle = new ClpTableLayoutHandle(
128141
new ClpTableHandle(new SchemaTableName(DEFAULT_SCHEMA_NAME, tableName), tablePath),
129142
Optional.empty(),
130143
metadataSql);
131-
List<ArchivesTableRow> expectedSplits = expectedSplitIds.stream()
132-
.map(expectedSplitId -> entry.getValue().get(expectedSplitId))
144+
List<String> expectedSplitPaths = expectedSplitIndexes.stream()
145+
.map(expectedSplitIndex -> format("%s/%s", tablePath, entry.getValue().getIds().get(expectedSplitIndex)))
133146
.collect(toImmutableList());
134147
List<ClpSplit> actualSplits = clpSplitProvider.listSplits(layoutHandle);
135-
assertEquals(actualSplits.size(), expectedSplits.size());
148+
assertEquals(actualSplits.size(), expectedSplitPaths.size());
136149

137-
ImmutableSet<String> actualSplitPaths = actualSplits.stream()
150+
ImmutableList<String> actualSplitPaths = actualSplits.stream()
138151
.map(ClpSplit::getPath)
139-
.collect(toImmutableSet());
140-
141-
ImmutableSet<String> expectedSplitPaths = expectedSplits.stream()
142-
.map(split -> tablePath + "/" + split.getId())
143-
.collect(toImmutableSet());
152+
.collect(toImmutableList());
144153

145154
assertEquals(actualSplitPaths, expectedSplitPaths);
146155
}

‎presto-clp/src/test/java/com/facebook/presto/plugin/clp/mockdb/ClpMockMetadataDatabase.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,7 @@ public static final class Builder
217217
private Builder()
218218
{
219219
setDatabaseUrl(format("/tmp/%s", UUID.randomUUID()));
220+
setArchiveStorageDirectory("");
220221
setUsername(MOCK_METADATA_DB_DEFAULT_USERNAME);
221222
setPassword(MOCK_METADATA_DB_DEFAULT_PASSWORD);
222223
setTablePrefix(MOCK_METADATA_DB_DEFAULT_TABLE_PREFIX);

‎presto-clp/src/test/java/com/facebook/presto/plugin/clp/mockdb/table/ArchivesTableRows.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,4 +69,19 @@ public ArchivesTableRows(
6969
this.endTimestamps = endTimestamps;
7070
this.numberOfRows = ids.size();
7171
}
72+
73+
public List<String> getIds()
74+
{
75+
return ids;
76+
}
77+
78+
public List<Long> getBeginTimestamps()
79+
{
80+
return beginTimestamps;
81+
}
82+
83+
public List<Long> getEndTimestamps()
84+
{
85+
return endTimestamps;
86+
}
7287
}

0 commit comments

Comments
 (0)