Skip to content

Commit 1cdf004

Browse files
authored
Revert "[Delta] Support absolute path in Clone" (delta-io#4740)
1 parent adce7a4 commit 1cdf004

File tree

5 files changed

+4
-14
lines changed

5 files changed

+4
-14
lines changed

iceberg/src/main/scala/org/apache/spark/sql/delta/IcebergFileManifest.scala

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,6 @@ class IcebergFileManifest(
7979
_sizeInBytes.get
8080
}
8181

82-
override def supportAbsolutePath: Boolean = true
83-
8482
def allFiles: Dataset[ConvertTargetFile] = {
8583
if (fileSparkResults.isEmpty) fileSparkResults = Some(getFileSparkResults())
8684
fileSparkResults.get

spark/src/main/scala/org/apache/spark/sql/delta/commands/CloneTableCommand.scala

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -239,14 +239,11 @@ abstract class CloneConvertedSource(spark: SparkSession) extends CloneSource {
239239
val partitionSchema = convertTargetTable.partitionSchema
240240

241241
{
242-
val fileManifest = convertTargetTable.fileManifest
243-
val supportAbsolutePath = fileManifest.supportAbsolutePath
244-
fileManifest.allFiles.mapPartitions { targetFile =>
242+
convertTargetTable.fileManifest.allFiles.mapPartitions { targetFile =>
245243
val basePath = new Path(baseDir)
246244
val fs = basePath.getFileSystem(conf.value.value)
247245
targetFile.map(ConvertUtils.createAddFile(
248-
_, basePath, fs, SQLConf.get, Some(partitionSchema),
249-
supportAbsolutePath))
246+
_, basePath, fs, SQLConf.get, Some(partitionSchema)))
250247
}
251248
}
252249
}

spark/src/main/scala/org/apache/spark/sql/delta/commands/ConvertToDeltaCommand.scala

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -305,13 +305,10 @@ abstract class ConvertToDeltaCommandBase(
305305
val shouldCollectStats = collectStats && statsEnabled
306306
val statsBatchSize = conf.getConf(DeltaSQLConf.DELTA_IMPORT_BATCH_SIZE_STATS_COLLECTION)
307307
var numFiles = 0L
308-
val useAbsolutePath = manifest.supportAbsolutePath
309308
manifest.getFiles.grouped(statsBatchSize).flatMap { batch =>
310309
val adds = batch.map(
311310
ConvertUtils.createAddFile(
312-
_, txn.deltaLog.dataPath, fs, conf, Some(partitionSchema),
313-
useAbsolutePath || deltaPath.isDefined
314-
))
311+
_, txn.deltaLog.dataPath, fs, conf, Some(partitionSchema), deltaPath.isDefined))
315312
if (shouldCollectStats) {
316313
logInfo(
317314
log"Collecting stats for a batch of " +

spark/src/main/scala/org/apache/spark/sql/delta/commands/convert/ConvertUtils.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -221,7 +221,7 @@ trait ConvertUtilsBase extends DeltaLogging {
221221
s"Fail to relativize path $path against base path $basePath.")
222222
relativePath.toUri.toString
223223
} else {
224-
fs.makeQualified(path).toUri.toString
224+
path.toUri.toString
225225
}
226226

227227
AddFile(pathStrForAddFile, partition, file.length, file.modificationTime, dataChange = true,

spark/src/main/scala/org/apache/spark/sql/delta/commands/convert/interfaces.scala

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,6 @@ trait ConvertTargetFileManifest extends Closeable {
6767
/** Return the active files for a table in sequence */
6868
def getFiles: Iterator[ConvertTargetFile] = allFiles.toLocalIterator().asScala
6969

70-
/** All data file paths are required to be relative to table path when false */
71-
def supportAbsolutePath: Boolean = false
7270
/** Return the number of files for the table */
7371
def numFiles: Long = allFiles.count()
7472

0 commit comments

Comments
 (0)