Skip to content

Commit f7b9c77

Browse files
yruslanyruslan
committed
#38 Add support for Scala 2.13.
1 parent 8650644 commit f7b9c77

File tree

12 files changed

+215
-25
lines changed

12 files changed

+215
-25
lines changed

‎build.sbt‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,13 @@
1717
import Dependencies._
1818

1919
val scala211 = "2.11.12"
20-
val scala212 = "2.12.14"
20+
val scala212 = "2.12.18"
21+
val scala213 = "2.13.11"
2122

2223
ThisBuild / organization := "za.co.absa"
2324

24-
ThisBuild / scalaVersion := scala211
25-
ThisBuild / crossScalaVersions := Seq(scala211, scala212)
25+
ThisBuild / scalaVersion := scala212
26+
ThisBuild / crossScalaVersions := Seq(scala211, scala212, scala213)
2627

2728
ThisBuild / scalacOptions := Seq("-unchecked", "-deprecation")
2829

@@ -36,11 +37,21 @@ lazy val hats = (project in file("."))
3637
name := "spark-hats",
3738
printSparkVersion := {
3839
val log = streams.value.log
39-
log.info(s"Building with Spark $sparkVersion")
40-
sparkVersion
40+
val effectiveSparkVersion = sparkVersion(scalaVersion.value)
41+
log.info(s"Building with Spark $effectiveSparkVersion")
42+
effectiveSparkVersion
4143
},
42-
(Compile / compile) := ((Compile / compile) dependsOn printSparkVersion).value,
43-
libraryDependencies ++= SparkHatsDependencies :+ getScalaDependency(scalaVersion.value),
44+
Compile / compile := ((Compile / compile) dependsOn printSparkVersion).value,
45+
Compile / unmanagedSourceDirectories += {
46+
val sourceDir = (Compile / sourceDirectory).value
47+
CrossVersion.partialVersion(scalaVersion.value) match {
48+
case Some((2, n)) if n == 11 => sourceDir / "scala_2.11"
49+
case Some((2, n)) if n == 12 => sourceDir / "scala_2.12"
50+
case Some((2, n)) if n == 13 => sourceDir / "scala_2.13"
51+
case _ => throw new RuntimeException("Unsupported Scala version")
52+
}
53+
},
54+
libraryDependencies ++= getSparkHatsDependencies(scalaVersion.value) ++ getHofsDependency(scalaVersion.value) :+ getScalaDependency(scalaVersion.value),
4455
releasePublishArtifactsAction := PgpKeys.publishSigned.value,
4556
Test / fork := true
4657
).enablePlugins(AutomateHeaderPlugin)

‎project/Dependencies.scala‎

Lines changed: 28 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,24 +18,42 @@ import sbt._
1818

1919
object Dependencies {
2020

21-
def sparkVersion: String = sys.props.getOrElse("SPARK_VERSION", "2.4.8")
22-
private val sparkHofsVersion = "0.4.0"
21+
val defaultSparkVersionForScala211 = "2.4.8"
22+
val defaultSparkVersionForScala212 = "3.3.2"
23+
val defaultSparkVersionForScala213 = "3.4.1"
2324

24-
private val scalatestVersion = "3.0.3"
25+
private val sparkHofsVersion = "0.4.0"
26+
private val scalatestVersion = "3.2.14"
2527

2628
def getScalaDependency(scalaVersion: String): ModuleID = "org.scala-lang" % "scala-library" % scalaVersion % Provided
2729

28-
val SparkHatsDependencies: Seq[ModuleID] = Seq(
29-
// compile
30-
"za.co.absa" %% "spark-hofs" % sparkHofsVersion,
31-
30+
def getSparkHatsDependencies(scalaVersion: String): Seq[ModuleID] = Seq(
3231
// provided
33-
"org.apache.spark" %% "spark-core" % sparkVersion % Provided,
34-
"org.apache.spark" %% "spark-sql" % sparkVersion % Provided,
35-
"org.apache.spark" %% "spark-catalyst" % sparkVersion % Provided,
32+
"org.apache.spark" %% "spark-core" % sparkVersion(scalaVersion) % Provided,
33+
"org.apache.spark" %% "spark-sql" % sparkVersion(scalaVersion) % Provided,
34+
"org.apache.spark" %% "spark-catalyst" % sparkVersion(scalaVersion) % Provided,
3635

3736
// test
3837
"org.scalatest" %% "scalatest" % scalatestVersion % Test
3938
)
4039

40+
def getHofsDependency(scalaVersion: String): Seq[ModuleID] = if (scalaVersion.startsWith("2.11.")) {
41+
Seq("za.co.absa" %% "spark-hofs" % sparkHofsVersion)
42+
} else {
43+
Seq.empty
44+
}
45+
46+
def sparkVersion(scalaVersion: String): String = sys.props.getOrElse("SPARK_VERSION", sparkFallbackVersion(scalaVersion))
47+
48+
def sparkFallbackVersion(scalaVersion: String): String = {
49+
if (scalaVersion.startsWith("2.11.")) {
50+
defaultSparkVersionForScala211
51+
} else if (scalaVersion.startsWith("2.12.")) {
52+
defaultSparkVersionForScala212
53+
} else if (scalaVersion.startsWith("2.13.")) {
54+
defaultSparkVersionForScala213
55+
} else {
56+
throw new IllegalArgumentException(s"Scala $scalaVersion not supported.")
57+
}
58+
}
4159
}

‎project/build.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,4 @@
1313
# limitations under the License.
1414
#
1515

16-
sbt.version=1.6.2
16+
sbt.version=1.9.2

‎src/main/scala/za/co/absa/spark/hats/transformations/NestedArrayTransformations.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ package za.co.absa.spark.hats.transformations
1919
import org.apache.spark.sql.functions.{array, callUDF, col, flatten, struct, when, concat}
2020
import org.apache.spark.sql.types.{ArrayType, DataType, StructField, StructType}
2121
import org.apache.spark.sql.{Column, DataFrame}
22-
import za.co.absa.spark.hofs._
22+
import za.co.absa.spark.hats.HofsWrapper._
2323
import za.co.absa.spark.hats.utils.SchemaUtils
2424

2525
import scala.collection.mutable.ListBuffer

‎src/main/scala_2.11/za/co/absa/spark/hats/HofsWrapper.scala‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/*
2+
* Copyright 2020 ABSA Group Limited
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package za.co.absa.spark.hats
18+
19+
import org.apache.spark.sql.Column
20+
21+
import za.co.absa.spark.hofs.{transform => hofsTransform}
22+
23+
/**
24+
* This is a wrapper for high order functions depending on Scala version
25+
*/
26+
object HofsWrapper {
27+
/**
28+
* Applies the function `f` to every element in the `array`. The method is an equivalent to the `map` function
29+
* from functional programming.
30+
*
31+
* @param array A column of arrays
32+
* @param f A function transforming individual elements of the array
33+
* @param elementName The name of the lambda variable. The value is used in Spark execution plans.
34+
* @return A column of arrays with transformed elements
35+
*/
36+
def transform(
37+
array: Column,
38+
f: Column => Column,
39+
elementName: String): Column = {
40+
hofsTransform(array, f, elementName)
41+
}
42+
}

‎src/main/scala_2.12/za/co/absa/spark/hats/HofsWrapper.scala‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/*
2+
* Copyright 2020 ABSA Group Limited
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package za.co.absa.spark.hats
18+
19+
import org.apache.spark.sql.Column
20+
import org.apache.spark.sql.functions.{transform => sparkTransform}
21+
22+
/**
23+
* This is a wrapper for high order functions depending on Scala version
24+
*/
25+
object HofsWrapper {
26+
/**
27+
* Applies the function `f` to every element in the `array`. The method is an equivalent to the `map` function
28+
* from functional programming.
29+
*
30+
* @param array A column of arrays
31+
* @param f A function transforming individual elements of the array
32+
* @param elementName The name of the lambda variable. The value is used in Spark execution plans.
33+
* @return A column of arrays with transformed elements
34+
*/
35+
def transform(
36+
array: Column,
37+
f: Column => Column,
38+
elementName: String): Column = {
39+
sparkTransform(array, f)
40+
}
41+
}

‎src/main/scala_2.13/za/co/absa/spark/hats/HofsWrapper.scala‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* Copyright 2020 ABSA Group Limited
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package za.co.absa.spark.hats
18+
19+
import org.apache.spark.sql.{Column, functions => sparkTransform}
20+
21+
/**
22+
* This is a wrapper for high order functions depending on Scala version
23+
*/
24+
object HofsWrapper {
25+
/**
26+
* Applies the function `f` to every element in the `array`. The method is an equivalent to the `map` function
27+
* from functional programming.
28+
*
29+
* @param array A column of arrays
30+
* @param f A function transforming individual elements of the array
31+
* @param elementName The name of the lambda variable. The value is used in Spark execution plans.
32+
* @return A column of arrays with transformed elements
33+
*/
34+
def transform(
35+
array: Column,
36+
f: Column => Column,
37+
elementName: String): Column = {
38+
sparkTransform(array, f)
39+
}
40+
}

‎src/test/resources/log4j.properties‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Copyright 2020 ABSA Group Limited
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an "AS IS" BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
14+
log4j.rootCategory=INFO, console
15+
log4j.appender.console=org.apache.log4j.ConsoleAppender
16+
log4j.appender.console.target=System.err
17+
log4j.appender.console.layout=org.apache.log4j.PatternLayout
18+
log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
19+
log4j.appender.console.Threshold=ERROR

‎src/test/resources/log4j2.properties‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Copyright 2020 ABSA Group Limited
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an "AS IS" BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
14+
log4j.rootCategory=INFO, console
15+
log4j.appender.console=org.apache.log4j.ConsoleAppender
16+
log4j.appender.console.target=System.err
17+
log4j.appender.console.layout=org.apache.log4j.PatternLayout
18+
log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
19+
log4j.appender.console.Threshold=ERROR

‎src/test/scala/za/co/absa/spark/hats/transformations/DeepArrayErrorTransformationSuite.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,14 @@ package za.co.absa.spark.hats.transformations
1919
import org.apache.spark.sql.DataFrame
2020
import org.apache.spark.sql.functions._
2121
import org.apache.spark.sql.types.{IntegerType, StringType}
22-
import org.scalatest.FunSuite
22+
import org.scalatest.funsuite.AnyFunSuite
2323
import org.slf4j.LoggerFactory
2424
import za.co.absa.spark.hats.SparkTestBase
2525
import za.co.absa.spark.hats.transformations.samples.DeepArraySamples._
2626
import za.co.absa.spark.hats.transformations.samples.SampleErrorUDFs
2727
import za.co.absa.spark.hats.utils.JsonUtils
2828

29-
class DeepArrayErrorTransformationSuite extends FunSuite with SparkTestBase {
29+
class DeepArrayErrorTransformationSuite extends AnyFunSuite with SparkTestBase {
3030
// scalastyle:off line.size.limit
3131
// scalastyle:off null
3232

0 commit comments

Comments
 (0)