Skip to content

Commit 1e49109

Browse files
yew1ebyew1eb
committed
up
1 parent 9938629 commit 1e49109

File tree

19 files changed

+61
-38
lines changed

19 files changed

+61
-38
lines changed

‎auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronSQLAppStatusListener.scala‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@ import org.apache.spark.status.ElementTrackingStore
2525

2626
import org.apache.auron.spark.ui.AuronBuildInfoEvent
2727

28-
class AuronSQLAppStatusListener(@nowarn conf: SparkConf, kvstore: ElementTrackingStore)
28+
@nowarn("cat=unused") // conf temporarily unused
29+
class AuronSQLAppStatusListener(conf: SparkConf, kvstore: ElementTrackingStore)
2930
extends SparkListener
3031
with Logging {
3132

‎pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -374,6 +374,8 @@
374374
<target>${javaVersion}</target>
375375
<scalaVersion>${scalaLongVersion}</scalaVersion>
376376
<args>
377+
<arg>-deprecation</arg>
378+
<arg>-feature</arg>
377379
<arg>-Ywarn-unused</arg>
378380
<arg>-Xfatal-warnings</arg>
379381
</args>
@@ -835,12 +837,15 @@
835837
<artifactId>scala-maven-plugin</artifactId>
836838
<configuration>
837839
<args>
840+
<arg>-deprecation</arg>
841+
<arg>-feature</arg>
838842
<arg>-Ywarn-unused</arg>
839843
<!-- https://github.com/scalamacros/paradise is no longer actively developed. -->
840844
<!-- In Scala 2.13, the plugin's functionality has been included in the compiler -->
841845
<!-- directly under the -Ymacro-annotations flag. -->
842846
<arg>-Ymacro-annotations</arg>
843847
<arg>-Xfatal-warnings</arg>
848+
<arg>-Wconf:cat=deprecation:w</arg>
844849
</args>
845850
<compilerPlugins>
846851
<compilerPlugin>

‎spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/auron/ShimsImpl.scala‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package org.apache.spark.sql.auron
1919
import java.io.File
2020
import java.util.UUID
2121

22+
import scala.annotation.nowarn
2223
import scala.collection.mutable
2324

2425
import org.apache.commons.lang3.reflect.FieldUtils
@@ -962,6 +963,7 @@ class ShimsImpl extends Shims with Logging {
962963
}
963964
}
964965

966+
@nowarn("cat=unused") // Some params temporarily unused
965967
@sparkver("3.4 / 3.5")
966968
private def convertPromotePrecision(
967969
e: Expression,

‎spark-extension-shims-spark/src/main/scala/org/apache/spark/sql/execution/auron/shuffle/AuronRssShuffleManagerBase.scala‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,16 @@
1616
*/
1717
package org.apache.spark.sql.execution.auron.shuffle
1818

19+
import scala.annotation.nowarn
20+
1921
import org.apache.spark.{ShuffleDependency, SparkConf, TaskContext}
2022
import org.apache.spark.internal.Logging
2123
import org.apache.spark.shuffle._
2224
import org.apache.spark.sql.execution.auron.shuffle.AuronShuffleDependency.isArrowShuffle
2325

2426
import org.apache.auron.sparkver
2527

28+
@nowarn("cat=unused") // _conf temporarily unused
2629
abstract class AuronRssShuffleManagerBase(_conf: SparkConf) extends ShuffleManager with Logging {
2730
override def registerShuffle[K, V, C](
2831
shuffleId: Int,

‎spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronFunctionSuite.scala‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -278,8 +278,6 @@ class AuronFunctionSuite extends AuronQueryTest with BaseAuronSQLSuite {
278278
val dateTimeStampMin = format.parse(dateStringMin).getTime
279279
val dateTimeStampMax = format.parse(dateStringMax).getTime
280280
format = new SimpleDateFormat("yyyy-MM-dd")
281-
val dateString = "2015-01-01"
282-
val date = format.parse(dateString)
283281

284282
val functions =
285283
s"""
@@ -320,8 +318,6 @@ class AuronFunctionSuite extends AuronQueryTest with BaseAuronSQLSuite {
320318
val dateTimeStampMin = format.parse(dateStringMin).getTime
321319
val dateTimeStampMax = format.parse(dateStringMax).getTime
322320
format = new SimpleDateFormat("yyyy-MM-dd")
323-
val dateString = "2015-07-01"
324-
val date = format.parse(dateString)
325321

326322
val functions =
327323
s"""

‎spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronQuerySuite.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,7 @@ class AuronQuerySuite extends AuronQueryTest with BaseAuronSQLSuite with AuronSQ
209209
withTable("t") {
210210
sql(s"CREATE EXTERNAL TABLE t(c3 INT, c2 INT) USING ORC LOCATION '$path'")
211211

212-
val expected = if (forcePositionalEvolution) {
212+
val _ = if (forcePositionalEvolution) {
213213
correctAnswer
214214
} else {
215215
Seq(Row(null, 2), Row(null, 4), Row(null, 6), Row(null, null))
@@ -246,7 +246,7 @@ class AuronQuerySuite extends AuronQueryTest with BaseAuronSQLSuite with AuronSQ
246246
|LOCATION '$path'
247247
|""".stripMargin)
248248
sql("MSCK REPAIR TABLE t")
249-
val expected = if (forcePositionalEvolution) {
249+
val _ = if (forcePositionalEvolution) {
250250
correctAnswer
251251
} else {
252252
Seq(Row(null, 2, 1), Row(null, 4, 2), Row(null, 6, 3), Row(null, null, 4))

‎spark-extension/src/main/scala/org/apache/spark/sql/auron/AuronCallNativeWrapper.scala‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import java.nio.file.Files
2222
import java.nio.file.StandardCopyOption
2323
import java.util.concurrent.atomic.AtomicReference
2424

25+
import scala.annotation.nowarn
2526
import scala.collection.mutable.ArrayBuffer
2627

2728
import org.apache.arrow.c.ArrowArray
@@ -53,6 +54,7 @@ import org.apache.auron.protobuf.TaskDefinition
5354
* This class has been deprecated and migrated to {@link
5455
* org.apache.auron.jni.AuronCallNativeWrapper}. Will be removed in the future.
5556
*/
57+
@nowarn("cat=deprecation") // JniBridge is temporarily used (deprecated)
5658
@Deprecated
5759
case class AuronCallNativeWrapper(
5860
nativePlan: PhysicalPlanNode,
@@ -193,6 +195,7 @@ case class AuronCallNativeWrapper(
193195
}
194196
}
195197

198+
@nowarn("cat=deprecation") // JniBridge is temporarily used (deprecated)
196199
object AuronCallNativeWrapper extends Logging {
197200
def initNative(): Unit = {
198201
lazyInitNative

‎spark-extension/src/main/scala/org/apache/spark/sql/auron/AuronConverters.scala‎

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -17,16 +17,18 @@
1717
package org.apache.spark.sql.auron
1818

1919
import java.util.ServiceLoader
20+
2021
import scala.annotation.{nowarn, tailrec}
2122
import scala.collection.JavaConverters._
2223
import scala.collection.mutable
24+
2325
import org.apache.commons.lang3.reflect.MethodUtils
2426
import org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat
2527
import org.apache.spark.Partition
2628
import org.apache.spark.broadcast.Broadcast
27-
import org.apache.spark.internal.{Logging, config}
28-
import org.apache.spark.sql.auron.AuronConvertStrategy.{childOrderingRequiredTag, convertStrategyTag, convertToNonNativeTag, convertibleTag, isNeverConvert, joinSmallerSideTag, neverConvertReasonTag}
29-
import org.apache.spark.sql.auron.NativeConverters.{StubExpr, existTimestampType, isTypeSupported, roundRobinTypeSupported}
29+
import org.apache.spark.internal.{config, Logging}
30+
import org.apache.spark.sql.auron.AuronConvertStrategy.{childOrderingRequiredTag, convertibleTag, convertStrategyTag, convertToNonNativeTag, isNeverConvert, joinSmallerSideTag, neverConvertReasonTag}
31+
import org.apache.spark.sql.auron.NativeConverters.{existTimestampType, isTypeSupported, roundRobinTypeSupported, StubExpr}
3032
import org.apache.spark.sql.auron.util.AuronLogUtils.logDebugPlanConversion
3133
import org.apache.spark.sql.catalyst.expressions.AggregateWindowFunction
3234
import org.apache.spark.sql.catalyst.expressions.Alias
@@ -43,6 +45,7 @@ import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateExpression
4345
import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateFunction
4446
import org.apache.spark.sql.catalyst.expressions.aggregate.Final
4547
import org.apache.spark.sql.catalyst.expressions.aggregate.Partial
48+
import org.apache.spark.sql.catalyst.optimizer.{BuildLeft, BuildRight}
4649
import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning
4750
import org.apache.spark.sql.catalyst.plans.physical.Partitioning
4851
import org.apache.spark.sql.catalyst.plans.physical.RangePartitioning
@@ -70,6 +73,7 @@ import org.apache.spark.sql.hive.execution.InsertIntoHiveTable
7073
import org.apache.spark.sql.hive.execution.auron.plan.NativeHiveTableScanBase
7174
import org.apache.spark.sql.internal.SQLConf
7275
import org.apache.spark.sql.types.LongType
76+
7377
import org.apache.auron.configuration.AuronConfiguration
7478
import org.apache.auron.jni.AuronAdaptor
7579
import org.apache.auron.metric.SparkMetricNode
@@ -153,15 +157,6 @@ object AuronConverters extends Logging {
153157
supportedShuffleManagers.exists(name.contains)
154158
}
155159

156-
// format: off
157-
// scalafix:off
158-
// necessary imports for cross spark versions build
159-
@nowarn("cat=unused-imports")
160-
import org.apache.spark.sql.catalyst.plans._
161-
import org.apache.spark.sql.catalyst.optimizer._
162-
// scalafix:on
163-
// format: on
164-
165160
def convertSparkPlanRecursively(exec: SparkPlan): SparkPlan = {
166161
// convert
167162
var danglingConverted: Seq[SparkPlan] = Nil
@@ -425,12 +420,14 @@ object AuronConverters extends Logging {
425420
@sparkver(" 3.2 / 3.3 / 3.4 / 3.5")
426421
def getIsSkewJoinFromSHJ(exec: ShuffledHashJoinExec): Boolean = exec.isSkewJoin
427422

423+
@nowarn("cat=unused")
428424
@sparkver("3.0 / 3.1")
429425
def getIsSkewJoinFromSHJ(exec: ShuffledHashJoinExec): Boolean = false
430426

431427
@sparkver("3.1 / 3.2 / 3.3 / 3.4 / 3.5")
432428
def getShuffleOrigin(exec: ShuffleExchangeExec): Option[Any] = Some(exec.shuffleOrigin)
433429

430+
@nowarn("cat=unused")
434431
@sparkver("3.0")
435432
def getShuffleOrigin(exec: ShuffleExchangeExec): Option[Any] = None
436433

@@ -552,7 +549,7 @@ object AuronConverters extends Logging {
552549
org.apache.spark.sql.execution.auron.plan.BuildLeft
553550
case Some(org.apache.spark.sql.execution.auron.plan.BuildRight) =>
554551
org.apache.spark.sql.execution.auron.plan.BuildRight
555-
case None =>
552+
case _ =>
556553
logWarning("JoinSmallerSideTag is missing, defaults to BuildRight")
557554
org.apache.spark.sql.execution.auron.plan.BuildRight
558555
}

‎spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeConverters.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -993,12 +993,12 @@ object NativeConverters extends Logging {
993993
val children = e.children.map(Cast(_, e.dataType))
994994
buildScalarFunction(pb.ScalarFunction.Coalesce, children, e.dataType)
995995

996-
case e @ StringLPad(str, len, pad) =>
996+
case _ @StringLPad(str, len, pad) =>
997997
buildScalarFunction(
998998
pb.ScalarFunction.Lpad,
999999
Seq(str, castIfNecessary(len, LongType), pad),
10001000
StringType)
1001-
case e @ StringRPad(str, len, pad) =>
1001+
case _ @StringRPad(str, len, pad) =>
10021002
buildScalarFunction(
10031003
pb.ScalarFunction.Rpad,
10041004
Seq(str, castIfNecessary(len, LongType), pad),

‎spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeHelper.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ object NativeHelper extends Logging {
100100
if (nativePlan == null) {
101101
return Iterator.empty
102102
}
103-
var auronCallNativeWrapper = new org.apache.auron.jni.AuronCallNativeWrapper(
103+
val auronCallNativeWrapper = new org.apache.auron.jni.AuronCallNativeWrapper(
104104
ROOT_ALLOCATOR,
105105
nativePlan,
106106
metrics,

0 commit comments

Comments
 (0)