Skip to content

Commit 550e448

Browse files
wraymowraymo
committed
1. Remove kqlQuery and schemaTableName from ClpSplit, rename archivePath to splitPath
2. Fix a bug with array type annotation in ClpSchemaTree 3. Update table schema for ClpMySqlSplitProvider and ClpMySqlMetadataProvider and update tests accordingly
1 parent e7b3bc6 commit 550e448

File tree

6 files changed

+35
-55
lines changed

6 files changed

+35
-55
lines changed

‎presto-clp/src/main/java/com/facebook/presto/plugin/clp/ClpSplit.java‎

Lines changed: 5 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
import com.facebook.presto.spi.ConnectorSplit;
1717
import com.facebook.presto.spi.HostAddress;
1818
import com.facebook.presto.spi.NodeProvider;
19-
import com.facebook.presto.spi.SchemaTableName;
2019
import com.facebook.presto.spi.schedule.NodeSelectionStrategy;
2120
import com.fasterxml.jackson.annotation.JsonCreator;
2221
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -25,44 +24,24 @@
2524
import javax.annotation.Nullable;
2625

2726
import java.util.List;
28-
import java.util.Optional;
2927

3028
import static com.facebook.presto.spi.schedule.NodeSelectionStrategy.NO_PREFERENCE;
3129

3230
public class ClpSplit
3331
implements ConnectorSplit
3432
{
35-
private final SchemaTableName schemaTableName;
36-
private final String archivePath;
37-
private final Optional<String> kqlQuery;
33+
private final String splitPath;
3834

3935
@JsonCreator
40-
public ClpSplit(@JsonProperty("schemaTableName") @Nullable SchemaTableName schemaTableName,
41-
@JsonProperty("archivePath") @Nullable String archivePath,
42-
@JsonProperty("kqlQuery") Optional<String> kqlQuery)
36+
public ClpSplit(@JsonProperty("splitPath") @Nullable String splitPath)
4337
{
44-
this.schemaTableName = schemaTableName;
45-
this.archivePath = archivePath;
46-
this.kqlQuery = kqlQuery;
38+
this.splitPath = splitPath;
4739
}
4840

4941
@JsonProperty
50-
@Nullable
51-
public SchemaTableName getSchemaTableName()
42+
public String getSplitPath()
5243
{
53-
return schemaTableName;
54-
}
55-
56-
@JsonProperty
57-
public String getArchivePath()
58-
{
59-
return archivePath;
60-
}
61-
62-
@JsonProperty
63-
public Optional<String> getKqlQuery()
64-
{
65-
return kqlQuery;
44+
return splitPath;
6645
}
6746

6847
@Override

‎presto-clp/src/main/java/com/facebook/presto/plugin/clp/metadata/ClpMySqlMetadataProvider.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,10 @@ public class ClpMySqlMetadataProvider
3434
{
3535
private static final Logger log = Logger.get(ClpMySqlMetadataProvider.class);
3636

37-
public static final String COLUMN_METADATA_PREFIX = "column_metadata_";
38-
private static final String QUERY_SELECT_COLUMNS = "SELECT * FROM %s" + COLUMN_METADATA_PREFIX + "%s";
39-
private static final String TABLE_METADATA_TABLE_SUFFIX = "table_metadata";
40-
private static final String QUERY_SELECT_TABLES = "SELECT table_name FROM %s" + TABLE_METADATA_TABLE_SUFFIX;
37+
public static final String COLUMN_METADATA_TABLE_SUFFIX = "_column_metadata";
38+
private static final String QUERY_SELECT_COLUMNS = "SELECT * FROM %s%s" + COLUMN_METADATA_TABLE_SUFFIX;
39+
private static final String DATASETS_TABLE_SUFFIX = "datasets";
40+
private static final String QUERY_SELECT_TABLES = "SELECT name FROM %s" + DATASETS_TABLE_SUFFIX;
4141

4242
private final ClpConfig config;
4343

@@ -98,7 +98,7 @@ public List<String> listTableNames(String schema)
9898
Statement statement = connection.createStatement();
9999
ResultSet resultSet = statement.executeQuery(query)) {
100100
while (resultSet.next()) {
101-
String tableName = resultSet.getString("table_name");
101+
String tableName = resultSet.getString("name");
102102
if (isValidIdentifier(tableName)) {
103103
tableNames.add(tableName);
104104
}

‎presto-clp/src/main/java/com/facebook/presto/plugin/clp/metadata/ClpSchemaTree.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,9 @@ public void addColumn(String fullName, byte type)
106106
ClpNode existing = current.children.get(leafName);
107107

108108
if (existing.type != null && !existing.type.equals(prestoType)) {
109-
String existingSuffix = existing.type.getDisplayName();
109+
String existingSuffix = (existing.type instanceof ArrayType)
110+
? "array"
111+
: existing.type.getDisplayName();
110112
String renamedExisting = leafName + "_" + existingSuffix;
111113

112114
current.children.remove(leafName);

‎presto-clp/src/main/java/com/facebook/presto/plugin/clp/split/ClpMySqlSplitProvider.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,9 @@ public class ClpMySqlSplitProvider
3636
private static final Logger log = Logger.get(ClpMySqlSplitProvider.class);
3737

3838
private static final String ARCHIVE_TABLE_SUFFIX = "_archives";
39-
private static final String TABLE_METADATA_TABLE_SUFFIX = "table_metadata";
40-
private static final String QUERY_SELECT_ARCHIVE_IDS = "SELECT archive_id FROM %s%s" + ARCHIVE_TABLE_SUFFIX;
41-
private static final String QUERY_SELECT_TABLE_METADATA = "SELECT table_path FROM %s" + TABLE_METADATA_TABLE_SUFFIX + " WHERE table_name = '%s'";
39+
private static final String DATASETS_TABLE_SUFFIX = "datasets";
40+
private static final String QUERY_SELECT_ARCHIVE_IDS = "SELECT id FROM %s%s" + ARCHIVE_TABLE_SUFFIX;
41+
private static final String QUERY_SELECT_TABLE_METADATA = "SELECT archive_storage_directory FROM %s" + DATASETS_TABLE_SUFFIX + " WHERE name = '%s'";
4242

4343
private final ClpConfig config;
4444

@@ -84,7 +84,7 @@ public List<ClpSplit> listSplits(ClpTableLayoutHandle clpTableLayoutHandle)
8484
log.error("Table metadata not found for table: %s", tableName);
8585
return ImmutableList.of();
8686
}
87-
tablePath = resultSet.getString("table_path");
87+
tablePath = resultSet.getString("archive_storage_directory");
8888
}
8989

9090
if (tablePath == null || tablePath.isEmpty()) {
@@ -96,9 +96,9 @@ public List<ClpSplit> listSplits(ClpTableLayoutHandle clpTableLayoutHandle)
9696
try (PreparedStatement statement = connection.prepareStatement(archivePathQuery);
9797
ResultSet resultSet = statement.executeQuery()) {
9898
while (resultSet.next()) {
99-
final String archiveId = resultSet.getString("archive_id");
99+
final String archiveId = resultSet.getString("id");
100100
final String archivePath = tablePath + "/" + archiveId;
101-
splits.add(new ClpSplit(tableSchemaName, archivePath, clpTableLayoutHandle.getKqlQuery()));
101+
splits.add(new ClpSplit(archivePath));
102102
}
103103
}
104104
}

‎presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpMetadata.java‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -61,8 +61,8 @@ public void setUp()
6161
final String metadataDbUser = "sa";
6262
final String metadataDbPassword = "";
6363
final String metadataDbTablePrefix = "clp_";
64-
final String columnMetadataTablePrefix = "column_metadata_";
65-
final String tableMetadataSuffix = "table_metadata";
64+
final String columnMetadataTableSuffix = "_column_metadata";
65+
final String datasetsTableSuffix = "datasets";
6666

6767
ClpConfig config = new ClpConfig().setPolymorphicTypeEnabled(true)
6868
.setMetadataDbUrl(metadataDbUrl)
@@ -72,13 +72,13 @@ public void setUp()
7272
ClpMetadataProvider metadataProvider = new ClpMySqlMetadataProvider(config);
7373
metadata = new ClpMetadata(config, metadataProvider);
7474

75-
final String tableMetadataTableName = metadataDbTablePrefix + tableMetadataSuffix;
76-
final String columnMetadataTableName = metadataDbTablePrefix + columnMetadataTablePrefix + TABLE_NAME;
75+
final String datasetsTableName = metadataDbTablePrefix + datasetsTableSuffix;
76+
final String columnMetadataTableName = metadataDbTablePrefix + TABLE_NAME + columnMetadataTableSuffix;
7777

7878
final String createTableMetadataSQL = String.format(
7979
"CREATE TABLE IF NOT EXISTS %s (" +
80-
" table_name VARCHAR(512) PRIMARY KEY," +
81-
" table_path VARCHAR(1024) NOT NULL)", tableMetadataTableName);
80+
" name VARCHAR(255) PRIMARY KEY," +
81+
" archive_storage_directory VARCHAR(4096) NOT NULL)", datasetsTableName);
8282

8383
final String createColumnMetadataSQL = String.format(
8484
"CREATE TABLE IF NOT EXISTS %s (" +
@@ -87,7 +87,7 @@ public void setUp()
8787
" PRIMARY KEY (name, type))", columnMetadataTableName);
8888

8989
final String insertTableMetadataSQL = String.format(
90-
"INSERT INTO %s (table_name, table_path) VALUES (?, ?)", tableMetadataTableName);
90+
"INSERT INTO %s (name, archive_storage_directory) VALUES (?, ?)", datasetsTableName);
9191

9292
final String insertColumnMetadataSQL = String.format(
9393
"INSERT INTO %s (name, type) VALUES (?, ?)", columnMetadataTableName);

‎presto-clp/src/test/java/com/facebook/presto/plugin/clp/TestClpSplit.java‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ public void setUp()
5151
final String metadataDbUser = "sa";
5252
final String metadataDbPassword = "";
5353
final String metadataDbTablePrefix = "clp_";
54-
final String tableMetadataSuffix = "table_metadata";
54+
final String datasetsTableSuffix = "datasets";
5555
final String archiveTableSuffix = "_archives";
5656

5757
this.config = new ClpConfig().setPolymorphicTypeEnabled(true)
@@ -60,20 +60,20 @@ public void setUp()
6060
.setMetadataDbPassword("")
6161
.setMetadataTablePrefix(metadataDbTablePrefix);
6262

63-
final String tableMetadataTableName = metadataDbTablePrefix + tableMetadataSuffix;
63+
final String datasetsTableName = metadataDbTablePrefix + datasetsTableSuffix;
6464
final String archiveTableFormat = metadataDbTablePrefix + "%s" + archiveTableSuffix;
6565

6666
final String createTableMetadataSQL = String.format(
6767
"CREATE TABLE IF NOT EXISTS %s (" +
68-
" table_name VARCHAR(512) PRIMARY KEY," +
69-
" table_path VARCHAR(1024) NOT NULL)", tableMetadataTableName);
68+
" name VARCHAR(255) PRIMARY KEY," +
69+
" archive_storage_directory VARCHAR(4096) NOT NULL)", datasetsTableName);
7070

7171
try (Connection conn = DriverManager.getConnection(metadataDbUrl, metadataDbUser, metadataDbPassword);
7272
Statement stmt = conn.createStatement()) {
7373
stmt.execute(createTableMetadataSQL);
7474

7575
// Insert table metadata in batch
76-
String insertTableMetadataSQL = String.format("INSERT INTO %s (table_name, table_path) VALUES (?, ?)", tableMetadataTableName);
76+
String insertTableMetadataSQL = String.format("INSERT INTO %s (name, archive_storage_directory) VALUES (?, ?)", datasetsTableName);
7777
try (PreparedStatement pstmt = conn.prepareStatement(insertTableMetadataSQL)) {
7878
for (String tableName : TABLE_NAME_LIST) {
7979
pstmt.setString(1, tableName);
@@ -88,13 +88,13 @@ public void setUp()
8888
String archiveTableName = String.format(archiveTableFormat, tableName);
8989
String createArchiveTableSQL = String.format(
9090
"CREATE TABLE IF NOT EXISTS %s (" +
91-
"id BIGINT AUTO_INCREMENT PRIMARY KEY, " +
92-
"archive_id VARCHAR(128) NOT NULL" +
91+
"pagination_id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY, " +
92+
"id VARCHAR(64) NOT NULL" +
9393
")",
9494
archiveTableName);
9595
stmt.execute(createArchiveTableSQL);
9696

97-
String insertArchiveTableSQL = String.format("INSERT INTO %s (archive_id) VALUES (?)", archiveTableName);
97+
String insertArchiveTableSQL = String.format("INSERT INTO %s (id) VALUES (?)", archiveTableName);
9898
try (PreparedStatement pstmt = conn.prepareStatement(insertArchiveTableSQL)) {
9999
for (int i = 0; i < NUM_SPLITS; i++) {
100100
pstmt.setString(1, "id_" + i);
@@ -132,8 +132,7 @@ public void testListSplits()
132132
List<ClpSplit> splits = splitProvider.listSplits(layoutHandle);
133133
assertEquals(splits.size(), NUM_SPLITS);
134134
for (int i = 0; i < NUM_SPLITS; i++) {
135-
assertEquals(splits.get(i).getArchivePath(), "/tmp/archives/" + tableName + "/id_" + i);
136-
assertEquals(splits.get(i).getKqlQuery(), Optional.empty());
135+
assertEquals(splits.get(i).getSplitPath(), "/tmp/archives/" + tableName + "/id_" + i);
137136
}
138137
}
139138
}

0 commit comments

Comments
 (0)