Bugfix/missing sources

build.sbt

@@ -50,32 +50,22 @@ lazy val commonJacocoExcludes: Seq[String] = Seq(
 
 lazy val parent = (project in file("."))
   .aggregate(
-    sparkCommonsSpark2.projectRefs ++
-      sparkCommonsSpark3.projectRefs ++
+    sparkCommons.projectRefs ++
       sparkCommonsTest.projectRefs: _*
   )
   .settings(
     name := "spark-commons-parent",
     publish / skip := true
   )
 
-lazy val sparkCommonsSpark2 = (projectMatrix in file("scala-spark2.4-jvm"))
+lazy val sparkCommons = (projectMatrix in file("spark-commons"))
   .settings(commonSettings: _*)
   .sparkRow(SparkVersionAxis(spark2), scalaVersions = Seq(scala211, scala212))
-  .settings(
-    Compile / unmanagedSourceDirectories := Seq((Compile / sourceDirectory).value / "main" / "scala")
-  )
-
-lazy val spark3Versions = Seq(spark32, spark33, spark34, spark35)
-lazy val sparkCommonsSpark3 = (projectMatrix in file("scala-spark3-jvm"))
-  .settings(commonSettings: _*)
   .sparkRow(SparkVersionAxis(spark32), scalaVersions = Seq(scala212, scala213))
   .sparkRow(SparkVersionAxis(spark33), scalaVersions = Seq(scala212, scala213))
   .sparkRow(SparkVersionAxis(spark34), scalaVersions = Seq(scala212, scala213))
   .sparkRow(SparkVersionAxis(spark35), scalaVersions = Seq(scala212, scala213))
-  .settings(
-    Compile / unmanagedSourceDirectories := Seq((Compile / sourceDirectory).value / "main" / "scala")
-  )
+  .dependsOn(sparkCommonsTest % "test")
 
 lazy val sparkCommonsTest = (projectMatrix in file("spark-commons-test"))
   .settings(

project/SparkVersionAxis.scala

@@ -20,37 +20,54 @@ import Dependencies._
 import JacocoSetup._
 import com.github.sbt.jacoco.JacocoKeys.{jacocoExcludes, jacocoReportSettings}
 
+import sbt._
+import sbt.Keys._
+import sbt.VirtualAxis._
+import sbt.internal.ProjectMatrix
+import sbtprojectmatrix.ProjectMatrixKeys._
+import Dependencies._
+
 case class SparkVersionAxis(sparkVersion: String) extends sbt.VirtualAxis.WeakAxis {
   val sparkVersionMinor: String = sparkVersion.split("\\.", 3).take(2).mkString(".")
-  override val directorySuffix = s"-spark${sparkVersionMinor}"
-  override val idSuffix: String = directorySuffix.replaceAll("""\W+""", "_")
+
+  // we have separate directory for Spark 2.4 (potentially other 2.x in the future)
+  // and a common one for all Spark 3.x versions
+  override val directorySuffix: String = if (sparkVersion.startsWith("2")) s"-spark${sparkVersionMinor}" else "-spark3"
+
+  // must be unique for all Spark 3.x versions in the matrix
+  override val idSuffix: String =
+    if (sparkVersion.startsWith("2")) directorySuffix.replaceAll("""\W+""", "_")
+    else s"-spark${sparkVersion.replaceAll("""\W+""", "_")}"
 }
 
+
 object SparkVersionAxis {
   private def camelCaseToLowerDashCase(origName: String): String = {
     origName
       .replaceAll("([A-Z])", "-$1")
       .toLowerCase()
   }
 
-  implicit class ProjectExtension(val projectMatrix: ProjectMatrix) extends AnyVal {
+  implicit class ProjectExtension(val p: ProjectMatrix) extends AnyVal {
 
     def sparkRow(sparkAxis: SparkVersionAxis, scalaVersions: Seq[String], settings: Def.SettingsDefinition*): ProjectMatrix = {
-      val sparkVersion = sparkAxis.sparkVersion
-      scalaVersions.foldLeft(projectMatrix)((currentProjectMatrix, scalaVersion) =>
-        currentProjectMatrix.customRow(
-          scalaVersions = Seq(scalaVersion),
-          axisValues = Seq(sparkAxis, VirtualAxis.jvm),
-          _.settings(
-            moduleName := camelCaseToLowerDashCase(
-              name.value.replaceAll("(Spark2|Spark3)$", "") + sparkAxis.directorySuffix
-            ),
-            libraryDependencies ++= sparkCommonsDependencies(sparkAxis.sparkVersion),
-            jacocoReportSettings := jacocoSettings(sparkVersion, scalaVersion),
-            jacocoExcludes := jacocoProjectExcludes(sparkVersion, scalaVersion)
-          ).settings(settings: _*)
-        )
+      p.customRow(
+        scalaVersions = scalaVersions,
+        axisValues = Seq(sparkAxis, VirtualAxis.jvm),
+        _.settings(
+          // must be defined to avoid conflicting target paths for individual spark versions' builds
+          target := (ThisBuild / baseDirectory).value / "target" / s"${camelCaseToLowerDashCase(name.value)}${sparkAxis.sparkVersion}-jvm-${scalaVersion.value.replaceAll("""\W+""", "_")}",
+          moduleName := {
+            val baseName = camelCaseToLowerDashCase(name.value)
+            if (sparkAxis.sparkVersion.startsWith("2"))
+              baseName + sparkAxis.directorySuffix
+            else
+              baseName + s"-spark${sparkAxis.sparkVersionMinor}"
+          },
+          libraryDependencies ++= sparkCommonsDependencies(sparkAxis.sparkVersion)
+        ).settings(settings: _*)
       )
     }
+
   }
 }