Skip to content

Commit 2c2a611

Browse files
committed
refactor, only one sources for spark3
1 parent e74bbd3 commit 2c2a611

File tree

8 files changed

+25
-110
lines changed

8 files changed

+25
-110
lines changed

build.sbt

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@ lazy val scala211 = "2.11.12"
1919
lazy val scala212 = "2.12.18"
2020
lazy val scala213 = "2.13.13"
2121
lazy val spark2 = "2.4.8"
22-
lazy val spark32 = "3.2.4"
23-
lazy val spark33 = "3.3.2"
24-
lazy val spark35 = "3.5.5"
22+
lazy val spark32 = "3.2.4"
23+
lazy val spark33 = "3.3.2"
24+
lazy val spark35 = "3.5.5"
2525

2626
import Dependencies._
2727
import SparkVersionAxis._
@@ -48,25 +48,40 @@ lazy val commonJacocoExcludes: Seq[String] = Seq(
4848
)
4949

5050
lazy val parent = (project in file("."))
51-
.aggregate(sparkCommons.projectRefs ++ sparkCommonsTest.projectRefs: _*)
51+
.aggregate(
52+
sparkCommonsSpark2.projectRefs ++
53+
sparkCommonsSpark3.projectRefs ++
54+
sparkCommonsTest.projectRefs: _*
55+
)
5256
.settings(
5357
name := "spark-commons-parent",
5458
publish / skip := true
5559
)
5660

57-
lazy val `sparkCommons` = (projectMatrix in file("spark-commons"))
61+
lazy val sparkCommonsSpark2 = (projectMatrix in file("scala-spark2.4-jvm"))
5862
.settings(commonSettings: _*)
5963
.sparkRow(SparkVersionAxis(spark2), scalaVersions = Seq(scala211, scala212))
64+
.settings(
65+
Compile / unmanagedSourceDirectories := Seq((Compile / sourceDirectory).value / "main" / "scala")
66+
)
67+
68+
lazy val spark3Versions = Seq(spark32, spark33, spark35)
69+
lazy val sparkCommonsSpark3 = (projectMatrix in file("scala-spark3-jvm"))
70+
.settings(commonSettings: _*)
6071
.sparkRow(SparkVersionAxis(spark32), scalaVersions = Seq(scala212, scala213))
6172
.sparkRow(SparkVersionAxis(spark33), scalaVersions = Seq(scala212, scala213))
6273
.sparkRow(SparkVersionAxis(spark35), scalaVersions = Seq(scala212, scala213))
63-
.dependsOn(sparkCommonsTest % "test")
74+
.settings(
75+
Compile / unmanagedSourceDirectories := Seq((Compile / sourceDirectory).value / "main" / "scala")
76+
)
6477

6578
lazy val sparkCommonsTest = (projectMatrix in file("spark-commons-test"))
6679
.settings(
6780
commonSettings ++ Seq(
6881
name := "spark-commons-test",
69-
libraryDependencies ++= sparkDependencies(if (scalaVersion.value == scala211) spark2 else spark35),
82+
libraryDependencies ++= sparkDependencies(
83+
if (scalaVersion.value == scala211) spark2 else spark35
84+
),
7085
Compile / unmanagedSourceDirectories += {
7186
val sourceDir = (Compile / sourceDirectory).value
7287
if (scalaVersion.value.startsWith("2.13")) {

project/SparkVersionAxis.scala

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,9 @@ object SparkVersionAxis {
4242
scalaVersions = Seq(scalaVersion),
4343
axisValues = Seq(sparkAxis, VirtualAxis.jvm),
4444
_.settings(
45-
moduleName := camelCaseToLowerDashCase(name.value + sparkAxis.directorySuffix),
45+
moduleName := camelCaseToLowerDashCase(
46+
name.value.replaceAll("(Spark2|Spark3)$", "") + sparkAxis.directorySuffix
47+
),
4648
libraryDependencies ++= sparkCommonsDependencies(sparkAxis.sparkVersion),
4749
jacocoReportSettings := jacocoSettings(sparkVersion, scalaVersion),
4850
jacocoExcludes := jacocoProjectExcludes(sparkVersion, scalaVersion)

spark-commons/src/main/scala-spark3.2-jvm/za/co/absa/spark/commons/adapters/CallUdfAdapter.scala renamed to spark-commons/src/main/scala-spark3-jvm/za/co/absa/spark/commons/adapters/CallUdfAdapter.scala

File renamed without changes.

spark-commons/src/main/scala-spark3.2-jvm/za/co/absa/spark/commons/adapters/TransformAdapter.scala renamed to spark-commons/src/main/scala-spark3-jvm/za/co/absa/spark/commons/adapters/TransformAdapter.scala

File renamed without changes.

spark-commons/src/main/scala-spark3.3-jvm/za/co/absa/spark/commons/adapters/CallUdfAdapter.scala

Lines changed: 0 additions & 24 deletions
This file was deleted.

spark-commons/src/main/scala-spark3.3-jvm/za/co/absa/spark/commons/adapters/TransformAdapter.scala

Lines changed: 0 additions & 27 deletions
This file was deleted.

spark-commons/src/main/scala-spark3.5-jvm/za/co/absa/spark/commons/adapters/CallUdfAdapter.scala

Lines changed: 0 additions & 24 deletions
This file was deleted.

spark-commons/src/main/scala-spark3.5-jvm/za/co/absa/spark/commons/adapters/TransformAdapter.scala

Lines changed: 0 additions & 27 deletions
This file was deleted.

0 commit comments

Comments
 (0)