Skip to content

Commit 0579d96

Browse files
evanvdiaPresto CUDF CI
evanvdia
authored and
Presto CUDF CI
committed
Add changes to populate the datasource metadata details
1 parent 2befaa4 commit 0579d96

File tree

11 files changed

+190
-39
lines changed

11 files changed

+190
-39
lines changed

‎presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcInputInfo.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
15+
package com.facebook.presto.plugin.jdbc;
16+
17+
import com.fasterxml.jackson.annotation.JsonProperty;
18+
19+
import static java.util.Objects.requireNonNull;
20+
21+
public class JdbcInputInfo
22+
{
23+
private final String tableLocation;
24+
25+
public JdbcInputInfo(
26+
@JsonProperty("tableLocation") String tableLocation)
27+
28+
{
29+
this.tableLocation = requireNonNull(tableLocation, "tableLocation is null");
30+
}
31+
32+
@JsonProperty
33+
public String getTableLocation()
34+
{
35+
return tableLocation;
36+
}
37+
}

‎presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadata.java‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,14 +55,15 @@ public class JdbcMetadata
5555
private final JdbcMetadataCache jdbcMetadataCache;
5656
private final JdbcClient jdbcClient;
5757
private final boolean allowDropTable;
58-
58+
private final String url;
5959
private final AtomicReference<Runnable> rollbackAction = new AtomicReference<>();
6060

61-
public JdbcMetadata(JdbcMetadataCache jdbcMetadataCache, JdbcClient jdbcClient, boolean allowDropTable)
61+
public JdbcMetadata(JdbcMetadataCache jdbcMetadataCache, JdbcClient jdbcClient, boolean allowDropTable, BaseJdbcConfig baseJdbcConfig)
6262
{
6363
this.jdbcMetadataCache = requireNonNull(jdbcMetadataCache, "jdbcMetadataCache is null");
6464
this.jdbcClient = requireNonNull(jdbcClient, "client is null");
6565
this.allowDropTable = allowDropTable;
66+
this.url = baseJdbcConfig.getConnectionUrl();
6667
}
6768

6869
@Override
@@ -189,7 +190,7 @@ public Optional<ConnectorOutputMetadata> finishCreateTable(ConnectorSession sess
189190
JdbcOutputTableHandle handle = (JdbcOutputTableHandle) tableHandle;
190191
jdbcClient.commitCreateTable(session, JdbcIdentity.from(session), handle);
191192
clearRollback();
192-
return Optional.empty();
193+
return Optional.of(new JdbcOutputMetadata(url));
193194
}
194195

195196
private void setRollback(Runnable action)
@@ -273,4 +274,10 @@ public String normalizeIdentifier(ConnectorSession session, String identifier)
273274
{
274275
return jdbcClient.normalizeIdentifier(session, identifier);
275276
}
277+
278+
@Override
279+
public Optional<Object> getInfo(ConnectorTableLayoutHandle tableHandle)
280+
{
281+
return Optional.of(new JdbcInputInfo(url));
282+
}
276283
}

‎presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcMetadataFactory.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,18 +22,20 @@ public class JdbcMetadataFactory
2222
private final JdbcMetadataCache jdbcMetadataCache;
2323
private final JdbcClient jdbcClient;
2424
private final boolean allowDropTable;
25+
private final BaseJdbcConfig baseJdbcConfig;
2526

2627
@Inject
27-
public JdbcMetadataFactory(JdbcMetadataCache jdbcMetadataCache, JdbcClient jdbcClient, JdbcMetadataConfig config)
28+
public JdbcMetadataFactory(JdbcMetadataCache jdbcMetadataCache, JdbcClient jdbcClient, JdbcMetadataConfig config, BaseJdbcConfig baseJdbcConfig)
2829
{
2930
this.jdbcMetadataCache = requireNonNull(jdbcMetadataCache, "jdbcMetadataCache is null");
3031
this.jdbcClient = requireNonNull(jdbcClient, "jdbcClient is null");
3132
requireNonNull(config, "config is null");
3233
this.allowDropTable = config.isAllowDropTable();
34+
this.baseJdbcConfig = requireNonNull(baseJdbcConfig, "baseJdbcConfig is null");
3335
}
3436

3537
public JdbcMetadata create()
3638
{
37-
return new JdbcMetadata(jdbcMetadataCache, jdbcClient, allowDropTable);
39+
return new JdbcMetadata(jdbcMetadataCache, jdbcClient, allowDropTable, baseJdbcConfig);
3840
}
3941
}

‎presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergWrittenPartitions.java‎ renamed to ‎presto-base-jdbc/src/main/java/com/facebook/presto/plugin/jdbc/JdbcOutputMetadata.java‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -11,31 +11,30 @@
1111
* See the License for the specific language governing permissions and
1212
* limitations under the License.
1313
*/
14-
package com.facebook.presto.iceberg;
14+
15+
package com.facebook.presto.plugin.jdbc;
1516

1617
import com.facebook.presto.spi.connector.ConnectorOutputMetadata;
1718
import com.fasterxml.jackson.annotation.JsonCreator;
1819
import com.fasterxml.jackson.annotation.JsonProperty;
19-
import com.google.common.collect.ImmutableList;
20-
21-
import java.util.List;
2220

2321
import static java.util.Objects.requireNonNull;
2422

25-
public class IcebergWrittenPartitions
23+
public class JdbcOutputMetadata
2624
implements ConnectorOutputMetadata
2725
{
28-
private final List<String> partitionNames;
26+
private final String tableLocation;
2927

3028
@JsonCreator
31-
public IcebergWrittenPartitions(@JsonProperty("partitionNames") List<String> partitionNames)
29+
public JdbcOutputMetadata(@JsonProperty("tableLocation") String tableLocation)
3230
{
33-
this.partitionNames = ImmutableList.copyOf(requireNonNull(partitionNames, "partitionNames is null"));
31+
this.tableLocation = requireNonNull(tableLocation, "tableLocation is null");
3432
}
3533

3634
@JsonProperty
37-
public List<String> getInfo()
35+
@Override
36+
public String getInfo()
3837
{
39-
return partitionNames;
38+
return tableLocation;
4039
}
4140
}

‎presto-base-jdbc/src/test/java/com/facebook/presto/plugin/jdbc/TestJdbcMetadata.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ public void setUp()
6666
database = new TestingDatabase();
6767
ListeningExecutorService executor = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("test-%s")));
6868
jdbcMetadataCache = new JdbcMetadataCache(executor, database.getJdbcClient(), new JdbcMetadataCacheStats(), OptionalLong.of(0), OptionalLong.of(0), 100);
69-
metadata = new JdbcMetadata(jdbcMetadataCache, database.getJdbcClient(), false);
69+
metadata = new JdbcMetadata(jdbcMetadataCache, database.getJdbcClient(), false, new BaseJdbcConfig());
7070
tableHandle = metadata.getTableHandle(SESSION, new SchemaTableName("example", "numbers"));
7171
}
7272

@@ -261,7 +261,7 @@ public void testDropTableTable()
261261
assertEquals(e.getErrorCode(), PERMISSION_DENIED.toErrorCode());
262262
}
263263

264-
metadata = new JdbcMetadata(jdbcMetadataCache, database.getJdbcClient(), true);
264+
metadata = new JdbcMetadata(jdbcMetadataCache, database.getJdbcClient(), true, new BaseJdbcConfig());
265265
metadata.dropTable(SESSION, tableHandle);
266266

267267
try {

‎presto-hive-common/src/main/java/com/facebook/presto/hive/HiveOutputInfo.java‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
15+
package com.facebook.presto.hive;
16+
17+
import com.fasterxml.jackson.annotation.JsonCreator;
18+
import com.fasterxml.jackson.annotation.JsonProperty;
19+
import com.google.common.collect.ImmutableList;
20+
21+
import java.util.List;
22+
import java.util.Objects;
23+
24+
import static java.util.Objects.requireNonNull;
25+
26+
public class HiveOutputInfo
27+
{
28+
private final List<String> partitionNames;
29+
private final String tableLocation;
30+
31+
@JsonCreator
32+
public HiveOutputInfo(
33+
@JsonProperty("partitionNames") List<String> partitionNames,
34+
@JsonProperty("tableLocation") String tableLocation)
35+
{
36+
this.partitionNames = ImmutableList.copyOf(requireNonNull(partitionNames, "partitionNames is null"));
37+
this.tableLocation = requireNonNull(tableLocation, "tableLocation is null");
38+
}
39+
40+
@JsonProperty
41+
public List<String> getPartitionNames()
42+
{
43+
return partitionNames;
44+
}
45+
46+
@JsonProperty
47+
public String getTableLocation()
48+
{
49+
return tableLocation;
50+
}
51+
52+
@Override
53+
public boolean equals(Object o)
54+
{
55+
if (o == null || getClass() != o.getClass()) {
56+
return false;
57+
}
58+
HiveOutputInfo that = (HiveOutputInfo) o;
59+
return Objects.equals(partitionNames, that.partitionNames) && Objects.equals(tableLocation, that.tableLocation);
60+
}
61+
62+
@Override
63+
public int hashCode()
64+
{
65+
return Objects.hash(partitionNames, tableLocation);
66+
}
67+
68+
@Override
69+
public String toString()
70+
{
71+
return "HiveOutputInfo{" +
72+
"partitionNames=" + partitionNames +
73+
", tableLocation='" + tableLocation + '\'' +
74+
'}';
75+
}
76+
}

‎presto-hive/src/main/java/com/facebook/presto/hive/HiveWrittenPartitions.java‎ renamed to ‎presto-hive-common/src/main/java/com/facebook/presto/hive/HiveOutputMetadata.java‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,26 +16,24 @@
1616
import com.facebook.presto.spi.connector.ConnectorOutputMetadata;
1717
import com.fasterxml.jackson.annotation.JsonCreator;
1818
import com.fasterxml.jackson.annotation.JsonProperty;
19-
import com.google.common.collect.ImmutableList;
20-
21-
import java.util.List;
2219

2320
import static java.util.Objects.requireNonNull;
2421

25-
public class HiveWrittenPartitions
22+
public class HiveOutputMetadata
2623
implements ConnectorOutputMetadata
2724
{
28-
private final List<String> partitionNames;
25+
private final HiveOutputInfo hiveOutputInfo;
2926

3027
@JsonCreator
31-
public HiveWrittenPartitions(@JsonProperty("partitionNames") List<String> partitionNames)
28+
public HiveOutputMetadata(@JsonProperty("hiveOutputInfo") HiveOutputInfo hiveOutputInfo)
3229
{
33-
this.partitionNames = ImmutableList.copyOf(requireNonNull(partitionNames, "partitionNames is null"));
30+
this.hiveOutputInfo = requireNonNull(hiveOutputInfo, "hiveOutputInfo is null");
3431
}
3532

3633
@JsonProperty
37-
public List<String> getInfo()
34+
@Override
35+
public HiveOutputInfo getInfo()
3836
{
39-
return partitionNames;
37+
return hiveOutputInfo;
4038
}
4139
}

‎presto-hive/src/main/java/com/facebook/presto/hive/HiveInputInfo.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,14 +24,17 @@ public class HiveInputInfo
2424
// Code that serialize HiveInputInfo into log would often need the ability to limit the length of log entries.
2525
// This boolean field allows such code to mark the log entry as length limited.
2626
private final boolean truncated;
27+
private final String tableLocation;
2728

2829
@JsonCreator
2930
public HiveInputInfo(
3031
@JsonProperty("partitionIds") List<String> partitionIds,
31-
@JsonProperty("truncated") boolean truncated)
32+
@JsonProperty("truncated") boolean truncated,
33+
@JsonProperty("tableLocation") String tableLocation)
3234
{
3335
this.partitionIds = partitionIds;
3436
this.truncated = truncated;
37+
this.tableLocation = tableLocation;
3538
}
3639

3740
@JsonProperty
@@ -45,4 +48,10 @@ public boolean isTruncated()
4548
{
4649
return truncated;
4750
}
51+
52+
@JsonProperty
53+
public String getTableLocation()
54+
{
55+
return tableLocation;
56+
}
4857
}

‎presto-hive/src/main/java/com/facebook/presto/hive/HiveMetadata.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -813,7 +813,7 @@ public Optional<Object> getInfo(ConnectorTableLayoutHandle layoutHandle)
813813
tableLayoutHandle.getPartitions().get().stream()
814814
.map(hivePartition -> hivePartition.getPartitionId().getPartitionName())
815815
.collect(toList()),
816-
false));
816+
false, tableLayoutHandle.getTablePath()));
817817
}
818818

819819
return Optional.empty();
@@ -1811,7 +1811,7 @@ public Optional<ConnectorOutputMetadata> finishCreateTable(ConnectorSession sess
18111811
metastore.createTable(session, table, principalPrivileges, Optional.of(writeInfo.getWritePath()), false, tableStatistics, emptyList());
18121812

18131813
if (handle.getPartitionedBy().isEmpty()) {
1814-
return Optional.of(new HiveWrittenPartitions(ImmutableList.of(UNPARTITIONED_ID.getPartitionName())));
1814+
return Optional.of(new HiveOutputMetadata(new HiveOutputInfo(ImmutableList.of(UNPARTITIONED_ID.getPartitionName()), writeInfo.getTargetPath().toString())));
18151815
}
18161816

18171817
if (isRespectTableFormat(session)) {
@@ -1840,10 +1840,10 @@ public Optional<ConnectorOutputMetadata> finishCreateTable(ConnectorSession sess
18401840
partitionStatistics);
18411841
}
18421842

1843-
return Optional.of(new HiveWrittenPartitions(
1843+
return Optional.of(new HiveOutputMetadata(new HiveOutputInfo(
18441844
partitionUpdates.stream()
18451845
.map(PartitionUpdate::getName)
1846-
.collect(toList())));
1846+
.collect(toList()), writeInfo.getTargetPath().toString())));
18471847
}
18481848

18491849
public static boolean shouldCreateFilesForMissingBuckets(Table table, ConnectorSession session)
@@ -2260,11 +2260,11 @@ else if (partitionUpdate.getUpdateMode() == NEW || partitionUpdate.getUpdateMode
22602260
}
22612261
}
22622262

2263-
return Optional.of(new HiveWrittenPartitions(
2263+
return Optional.of(new HiveOutputMetadata(new HiveOutputInfo(
22642264
partitionUpdates.stream()
22652265
.map(PartitionUpdate::getName)
22662266
.map(name -> name.isEmpty() ? UNPARTITIONED_ID.getPartitionName() : name)
2267-
.collect(toList())));
2267+
.collect(toList()), table.getStorage().getLocation())));
22682268
}
22692269

22702270
/**

‎presto-iceberg/src/main/java/com/facebook/presto/iceberg/IcebergAbstractMetadata.java‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,9 @@
2323
import com.facebook.presto.common.type.TimestampWithTimeZoneType;
2424
import com.facebook.presto.common.type.TypeManager;
2525
import com.facebook.presto.common.type.VarcharType;
26+
import com.facebook.presto.hive.HiveOutputInfo;
27+
import com.facebook.presto.hive.HiveOutputMetadata;
2628
import com.facebook.presto.hive.HivePartition;
27-
import com.facebook.presto.hive.HiveWrittenPartitions;
2829
import com.facebook.presto.hive.NodeVersion;
2930
import com.facebook.presto.iceberg.changelog.ChangelogOperation;
3031
import com.facebook.presto.iceberg.changelog.ChangelogUtil;
@@ -578,9 +579,9 @@ private Optional<ConnectorOutputMetadata> finishInsert(ConnectorSession session,
578579
throw new PrestoException(ICEBERG_COMMIT_ERROR, "Failed to commit Iceberg update to table: " + writableTableHandle.getTableName(), e);
579580
}
580581

581-
return Optional.of(new HiveWrittenPartitions(commitTasks.stream()
582+
return Optional.of(new HiveOutputMetadata(new HiveOutputInfo(commitTasks.stream()
582583
.map(CommitTaskData::getPath)
583-
.collect(toImmutableList())));
584+
.collect(toImmutableList()), icebergTable.location())));
584585
}
585586

586587
private Optional<ConnectorOutputMetadata> finishWrite(ConnectorSession session, IcebergWritableTableHandle writableTableHandle, Collection<Slice> fragments, ChangelogOperation operationType)
@@ -625,9 +626,9 @@ private Optional<ConnectorOutputMetadata> finishWrite(ConnectorSession session,
625626
throw new PrestoException(ICEBERG_COMMIT_ERROR, "Failed to commit Iceberg update to table: " + writableTableHandle.getTableName(), e);
626627
}
627628

628-
return Optional.of(new HiveWrittenPartitions(commitTasks.stream()
629+
return Optional.of(new HiveOutputMetadata(new HiveOutputInfo(commitTasks.stream()
629630
.map(CommitTaskData::getPath)
630-
.collect(toImmutableList())));
631+
.collect(toImmutableList()), icebergTable.location())));
631632
}
632633

633634
private void handleInsertTask(CommitTaskData task, Table icebergTable, AppendFiles appendFiles, ImmutableSet.Builder<String> writtenFiles)
@@ -1333,4 +1334,17 @@ protected Optional<String> getDataLocationBasedOnWarehouseDataDir(SchemaTableNam
13331334
{
13341335
return Optional.empty();
13351336
}
1337+
1338+
@Override
1339+
public Optional<Object> getInfo(ConnectorTableLayoutHandle tableHandle)
1340+
{
1341+
IcebergTableLayoutHandle icebergTableHandle = (IcebergTableLayoutHandle) tableHandle;
1342+
Optional<String> outputPath = icebergTableHandle.getTable().getOutputPath();
1343+
if (outputPath == null || !outputPath.isPresent()) {
1344+
return Optional.empty();
1345+
}
1346+
return Optional.of(new IcebergInputInfo(
1347+
icebergTableHandle.getTable().getIcebergTableName().getSnapshotId(),
1348+
outputPath.get()));
1349+
}
13361350
}

0 commit comments

Comments
 (0)