Skip to content

Commit e91b607

Browse files
committed
[SPARK-25918][SQL] LOAD DATA LOCAL INPATH should handle a relative path
## What changes were proposed in this pull request? Unfortunately, it seems that we missed this in 2.4.0. In Spark 2.4, if the default file system is not the local file system, `LOAD DATA LOCAL INPATH` only works in case of absolute paths. This PR aims to fix it to support relative paths. This is a regression in 2.4.0. ```scala $ ls kv1.txt kv1.txt scala> spark.sql("LOAD DATA LOCAL INPATH 'kv1.txt' INTO TABLE t") org.apache.spark.sql.AnalysisException: LOAD DATA input path does not exist: kv1.txt; ``` ## How was this patch tested? Pass the Jenkins Closes apache#22927 from dongjoon-hyun/SPARK-LOAD. Authored-by: Dongjoon Hyun <[email protected]> Signed-off-by: Dongjoon Hyun <[email protected]>
1 parent e9d3ca0 commit e91b607

File tree

2 files changed

+12
-2
lines changed

2 files changed

+12
-2
lines changed

sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -376,7 +376,8 @@ object LoadDataCommand {
376376
* @return qualified path object
377377
*/
378378
private[sql] def makeQualified(defaultUri: URI, workingDir: Path, path: Path): Path = {
379-
val pathUri = if (path.isAbsolute()) path.toUri() else new Path(workingDir, path).toUri()
379+
val newPath = new Path(workingDir, path)
380+
val pathUri = if (path.isAbsolute()) path.toUri() else newPath.toUri()
380381
if (pathUri.getScheme == null || pathUri.getAuthority == null &&
381382
defaultUri.getAuthority != null) {
382383
val scheme = if (pathUri.getScheme == null) defaultUri.getScheme else pathUri.getScheme
@@ -393,7 +394,7 @@ object LoadDataCommand {
393394
throw new IllegalArgumentException(e)
394395
}
395396
} else {
396-
path
397+
newPath
397398
}
398399
}
399400
}

sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,13 @@ package org.apache.spark.sql.hive.execution
2020
import java.io.File
2121

2222
import com.google.common.io.Files
23+
import org.apache.hadoop.fs.{FileContext, FsConstants, Path}
2324

2425
import org.apache.spark.sql.{AnalysisException, QueryTest, Row, SaveMode}
2526
import org.apache.spark.sql.catalyst.TableIdentifier
2627
import org.apache.spark.sql.catalyst.analysis.NoSuchTableException
2728
import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType}
29+
import org.apache.spark.sql.execution.command.LoadDataCommand
2830
import org.apache.spark.sql.hive.test.TestHiveSingleton
2931
import org.apache.spark.sql.test.SQLTestUtils
3032
import org.apache.spark.sql.types.StructType
@@ -439,4 +441,11 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
439441
}
440442
}
441443

444+
test("SPARK-25918: LOAD DATA LOCAL INPATH should handle a relative path") {
445+
val localFS = FileContext.getLocalFSFileContext()
446+
val workingDir = localFS.getWorkingDirectory
447+
val r = LoadDataCommand.makeQualified(
448+
FsConstants.LOCAL_FS_URI, workingDir, new Path("kv1.txt"))
449+
assert(r === new Path(s"$workingDir/kv1.txt"))
450+
}
442451
}

0 commit comments

Comments
 (0)