Merge pull request #48 from lyft/ctas

rohit-menon · web-flow · commit 13142bcd9348 · 2022-03-31T01:02:15.000+05:30
CTAS
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/InsertIntoHiveTable.scala
@@ -100,7 +100,7 @@ case class InsertIntoHiveTable(
       hiveQlTable.getMetadata
     )
     val tableLocation = hiveQlTable.getDataLocation
-    val tmpLocation = getExternalTmpPath(sparkSession, hadoopConf, tableLocation)
+    val tmpLocation = tableLocation
 
     try {
       processInsert(sparkSession, externalCatalog, hadoopConf, tableDesc, tmpLocation, child)
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/SaveAsHiveFile.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/SaveAsHiveFile.scala
@@ -84,6 +84,9 @@ private[hive] trait SaveAsHiveFile extends DataWritingCommand {
       jobId = java.util.UUID.randomUUID().toString,
       outputPath = outputLocation)
 
+    logDebug(s"Committer used before calling FileFormatWriter is '$committer'" +
+      s" and outputLocation is '$outputLocation'.")
+
     FileFormatWriter.write(
       sparkSession = sparkSession,
       plan = plan,

Original file line number	Diff line number	Diff line change
`@@ -100,7 +100,7 @@ case class InsertIntoHiveTable(`
`100`	`100`	`hiveQlTable.getMetadata`
`101`	`101`	`)`
`102`	`102`	`val tableLocation = hiveQlTable.getDataLocation`
`103`		`- val tmpLocation = getExternalTmpPath(sparkSession, hadoopConf, tableLocation)`
	`103`	`+ val tmpLocation = tableLocation`
`104`	`104`
`105`	`105`	`try {`
`106`	`106`	`processInsert(sparkSession, externalCatalog, hadoopConf, tableDesc, tmpLocation, child)`