Skip to content

Commit e62a6d8

Browse files
yew1ebyew1eb
committed
[AURON #1870][BUILD] Add -Xfatal-warnings to scala-maven-plugin and fix related compilation failures
1 parent a603cc0 commit e62a6d8

File tree

21 files changed

+59
-17
lines changed

21 files changed

+59
-17
lines changed

‎auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronSQLAppStatusListener.scala‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,16 @@
1616
*/
1717
package org.apache.spark.sql.execution.ui
1818

19+
import scala.annotation.nowarn
20+
1921
import org.apache.spark.{SparkConf, SparkContext}
2022
import org.apache.spark.internal.Logging
2123
import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent}
2224
import org.apache.spark.status.ElementTrackingStore
2325

2426
import org.apache.auron.spark.ui.AuronBuildInfoEvent
2527

28+
@nowarn("cat=unused") // conf temporarily unused
2629
class AuronSQLAppStatusListener(conf: SparkConf, kvstore: ElementTrackingStore)
2730
extends SparkListener
2831
with Logging {

‎pom.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -377,7 +377,10 @@
377377
<target>${javaVersion}</target>
378378
<scalaVersion>${scalaLongVersion}</scalaVersion>
379379
<args>
380+
<arg>-deprecation</arg>
381+
<arg>-feature</arg>
380382
<arg>-Ywarn-unused</arg>
383+
<arg>-Xfatal-warnings</arg>
381384
</args>
382385
</configuration>
383386
<dependencies>
@@ -862,11 +865,15 @@
862865
<artifactId>scala-maven-plugin</artifactId>
863866
<configuration>
864867
<args>
868+
<arg>-deprecation</arg>
869+
<arg>-feature</arg>
865870
<arg>-Ywarn-unused</arg>
866871
<!-- https://github.com/scalamacros/paradise is no longer actively developed. -->
867872
<!-- In Scala 2.13, the plugin's functionality has been included in the compiler -->
868873
<!-- directly under the -Ymacro-annotations flag. -->
869874
<arg>-Ymacro-annotations</arg>
875+
<arg>-Xfatal-warnings</arg>
876+
<arg>-Wconf:cat=deprecation:w</arg>
870877
</args>
871878
<compilerPlugins>
872879
<compilerPlugin>

‎spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/auron/ShimsImpl.scala‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package org.apache.spark.sql.auron
1919
import java.io.File
2020
import java.util.UUID
2121

22+
import scala.annotation.nowarn
2223
import scala.collection.mutable
2324

2425
import org.apache.commons.lang3.reflect.FieldUtils
@@ -965,6 +966,7 @@ class ShimsImpl extends Shims with Logging {
965966
}
966967
}
967968

969+
@nowarn("cat=unused") // Some params temporarily unused
968970
@sparkver("3.4 / 3.5")
969971
private def convertPromotePrecision(
970972
e: Expression,
@@ -997,6 +999,7 @@ class ShimsImpl extends Shims with Logging {
997999
}
9981000
}
9991001

1002+
@nowarn("cat=unused") // Some params temporarily unused
10001003
@sparkver("3.0 / 3.1 / 3.2")
10011004
private def convertBloomFilterAgg(agg: AggregateFunction): Option[pb.PhysicalAggExprNode] = None
10021005

@@ -1023,6 +1026,7 @@ class ShimsImpl extends Shims with Logging {
10231026
}
10241027
}
10251028

1029+
@nowarn("cat=unused") // Some params temporarily unused
10261030
@sparkver("3.0 / 3.1 / 3.2")
10271031
private def convertBloomFilterMightContain(
10281032
e: Expression,

‎spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/AuronBlockStoreShuffleReader.scala‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,9 @@ class AuronBlockStoreShuffleReader[K, C](
3737
extends AuronBlockStoreShuffleReaderBase[K, C](handle, context)
3838
with Logging {
3939

40+
// Touch mapOutputTracker to suppress -Xfatal-warnings (used in Spark 3.2+, unused in 3.0/3.1)
41+
private val _ = mapOutputTracker
42+
4043
override def readBlocks(): Iterator[InputStream] = {
4144
@sparkver("3.2 / 3.3 / 3.4 / 3.5")
4245
def fetchIterator = new ShuffleBlockFetcherIterator(

‎spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/AuronRssShuffleManagerBase.scala‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,16 @@
1616
*/
1717
package org.apache.spark.sql.execution.auron.shuffle
1818

19+
import scala.annotation.nowarn
20+
1921
import org.apache.spark.{ShuffleDependency, SparkConf, TaskContext}
2022
import org.apache.spark.internal.Logging
2123
import org.apache.spark.shuffle._
2224
import org.apache.spark.sql.execution.auron.shuffle.AuronShuffleDependency.isArrowShuffle
2325

2426
import org.apache.auron.sparkver
2527

28+
@nowarn("cat=unused") // _conf temporarily unused
2629
abstract class AuronRssShuffleManagerBase(_conf: SparkConf) extends ShuffleManager with Logging {
2730
override def registerShuffle[K, V, C](
2831
shuffleId: Int,

‎spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/execution/joins/auron/plan/NativeShuffledHashJoinExecProvider.scala‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
*/
1717
package org.apache.spark.sql.execution.joins.auron.plan
1818

19+
import scala.annotation.nowarn
20+
1921
import org.apache.spark.sql.auron.join.JoinBuildSides.JoinBuildSide
2022
import org.apache.spark.sql.catalyst.expressions.Expression
2123
import org.apache.spark.sql.catalyst.plans.JoinType
@@ -127,6 +129,7 @@ case object NativeShuffledHashJoinExecProvider {
127129
NativeShuffledHashJoinExec(left, right, leftKeys, rightKeys, joinType, buildSide)
128130
}
129131

132+
@nowarn("cat=unused") // Some params temporarily unused
130133
@sparkver("3.0")
131134
def provide(
132135
left: SparkPlan,

‎spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronFunctionSuite.scala‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -278,8 +278,6 @@ class AuronFunctionSuite extends AuronQueryTest with BaseAuronSQLSuite {
278278
val dateTimeStampMin = format.parse(dateStringMin).getTime
279279
val dateTimeStampMax = format.parse(dateStringMax).getTime
280280
format = new SimpleDateFormat("yyyy-MM-dd")
281-
val dateString = "2015-01-01"
282-
val date = format.parse(dateString)
283281

284282
val functions =
285283
s"""
@@ -320,8 +318,6 @@ class AuronFunctionSuite extends AuronQueryTest with BaseAuronSQLSuite {
320318
val dateTimeStampMin = format.parse(dateStringMin).getTime
321319
val dateTimeStampMax = format.parse(dateStringMax).getTime
322320
format = new SimpleDateFormat("yyyy-MM-dd")
323-
val dateString = "2015-07-01"
324-
val date = format.parse(dateString)
325321

326322
val functions =
327323
s"""

‎spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronQuerySuite.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -210,7 +210,7 @@ class AuronQuerySuite extends AuronQueryTest with BaseAuronSQLSuite with AuronSQ
210210
withTable("t") {
211211
sql(s"CREATE EXTERNAL TABLE t(c3 INT, c2 INT) USING ORC LOCATION '$path'")
212212

213-
val expected = if (forcePositionalEvolution) {
213+
val _ = if (forcePositionalEvolution) {
214214
correctAnswer
215215
} else {
216216
Seq(Row(null, 2), Row(null, 4), Row(null, 6), Row(null, null))
@@ -247,7 +247,7 @@ class AuronQuerySuite extends AuronQueryTest with BaseAuronSQLSuite with AuronSQ
247247
|LOCATION '$path'
248248
|""".stripMargin)
249249
sql("MSCK REPAIR TABLE t")
250-
if (forcePositionalEvolution) {
250+
val _ = if (forcePositionalEvolution) {
251251
correctAnswer
252252
} else {
253253
Seq(Row(null, 2, 1), Row(null, 4, 2), Row(null, 6, 3), Row(null, null, 4))

‎spark-extension/src/main/scala/org/apache/spark/sql/auron/AuronCallNativeWrapper.scala‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import java.nio.file.Files
2222
import java.nio.file.StandardCopyOption
2323
import java.util.concurrent.atomic.AtomicReference
2424

25+
import scala.annotation.nowarn
2526
import scala.collection.mutable.ArrayBuffer
2627

2728
import org.apache.arrow.c.ArrowArray
@@ -53,6 +54,7 @@ import org.apache.auron.protobuf.TaskDefinition
5354
* This class has been deprecated and migrated to {@link
5455
* org.apache.auron.jni.AuronCallNativeWrapper}. Will be removed in the future.
5556
*/
57+
@nowarn("cat=deprecation") // JniBridge is temporarily used (deprecated)
5658
@Deprecated
5759
case class AuronCallNativeWrapper(
5860
nativePlan: PhysicalPlanNode,
@@ -193,6 +195,7 @@ case class AuronCallNativeWrapper(
193195
}
194196
}
195197

198+
@nowarn("cat=deprecation") // JniBridge is temporarily used (deprecated)
196199
object AuronCallNativeWrapper extends Logging {
197200
def initNative(): Unit = {
198201
lazyInitNative

‎spark-extension/src/main/scala/org/apache/spark/sql/auron/AuronConverters.scala‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ package org.apache.spark.sql.auron
1818

1919
import java.util.ServiceLoader
2020

21-
import scala.annotation.tailrec
21+
import scala.annotation.{nowarn, tailrec}
2222
import scala.collection.JavaConverters._
2323
import scala.collection.mutable
2424

@@ -418,12 +418,14 @@ object AuronConverters extends Logging {
418418
@sparkver(" 3.2 / 3.3 / 3.4 / 3.5")
419419
def getIsSkewJoinFromSHJ(exec: ShuffledHashJoinExec): Boolean = exec.isSkewJoin
420420

421+
@nowarn("cat=unused")
421422
@sparkver("3.0 / 3.1")
422423
def getIsSkewJoinFromSHJ(exec: ShuffledHashJoinExec): Boolean = false
423424

424425
@sparkver("3.1 / 3.2 / 3.3 / 3.4 / 3.5")
425426
def getShuffleOrigin(exec: ShuffleExchangeExec): Option[Any] = Some(exec.shuffleOrigin)
426427

428+
@nowarn("cat=unused")
427429
@sparkver("3.0")
428430
def getShuffleOrigin(exec: ShuffleExchangeExec): Option[Any] = None
429431

@@ -649,6 +651,7 @@ object AuronConverters extends Logging {
649651
@sparkver("3.1 / 3.2 / 3.3 / 3.4 / 3.5")
650652
def isNullAwareAntiJoin(exec: BroadcastHashJoinExec): Boolean = exec.isNullAwareAntiJoin
651653

654+
@nowarn("cat=unused")
652655
@sparkver("3.0")
653656
def isNullAwareAntiJoin(exec: BroadcastHashJoinExec): Boolean = false
654657

0 commit comments

Comments
 (0)