[TOREE-556] Support Scala 2.13 (#218)

Support Scala 2.13

---------

Co-authored-by: Neil Skinner <[email protected]>

Author: pan3793

.github/workflows/build.yml

@@ -31,6 +31,7 @@ jobs:
       fail-fast: false
       matrix:
         java: [ '8', '11' ]
+        scala: [ '2.12', '2.13' ]
     env:
       # define Java options for both official sbt and sbt-extras
       JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
@@ -54,14 +55,14 @@ jobs:
       uses: sbt/setup-sbt@v1        
     - name: Build
       run: |
-        make clean release
+        make SCALA_VERSION=${{ matrix.scala }} clean release
     - name: Run tests
       run: |
-        make test
+        make SCALA_VERSION=${{ matrix.scala }} test
 #    See https://issues.apache.org/jira/browse/TOREE-526
 #    - name: Run system tests
 #      run: |
-#        make system-test
-    - name: Run license eudit
+#        make SCALA_VERSION=${{ matrix.scala }} system-test
+    - name: Run license audit
       run: |
-        make audit-licenses
+        make SCALA_VERSION=${{ matrix.scala }} audit-licenses

Dockerfile

@@ -27,7 +27,7 @@ RUN curl -sL https://deb.nodesource.com/setup_0.12 | bash - && \
 
 # for Apache Spark demos
 ENV APACHE_SPARK_VERSION 3.4.4
-ENV APACHE_SPARK_CUSTOM_NAME=hadoop3
+ARG SCALA_VERSION=2.12
 
 RUN apt-get -y update && \
     apt-get -y install software-properties-common
@@ -47,11 +47,11 @@ RUN echo "===> install Java"  && \
     update-java-alternatives -s java-8-oracle
 
 RUN cd /tmp && \
+    if [ "$SCALA_VERSION" = "2.13" ]; then APACHE_SPARK_CUSTOM_NAME=hadoop3-scala2.13; else APACHE_SPARK_CUSTOM_NAME=hadoop3; fi && \
     wget -q https://archive.apache.org/dist/spark/spark-${APACHE_SPARK_VERSION}/spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME}.tgz && \
     tar xzf spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME}.tgz -C /usr/local && \
-    rm spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME}.tgz
-
-RUN cd /usr/local && ln -s spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME} spark
+    rm spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME}.tgz && \
+    ln -snf /usr/local/spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME} /usr/local/spark
 
 # R support
 RUN apt-get update && \

Dockerfile.toree-dev

@@ -23,8 +23,8 @@ FROM jupyter/all-spark-notebook
 USER root
 
 # Spark dependencies
-ENV APACHE_SPARK_VERSION 3.3.2
-ENV APACHE_SPARK_CUSTOM_NAME=hadoop3
+ARG APACHE_SPARK_VERSION=3.4.4
+ARG SCALA_VERSION=2.12
 
 RUN apt-get -y update && \
     apt-get install -y --no-install-recommends openjdk-8-jdk ca-certificates-java  && \
@@ -36,14 +36,11 @@ RUN apt-get -y update && \
 
 # Installing Spark3
 RUN cd /tmp && \
+    if [ "$SCALA_VERSION" = "2.13" ]; then APACHE_SPARK_CUSTOM_NAME=hadoop3-scala2.13; else APACHE_SPARK_CUSTOM_NAME=hadoop3; fi && \
     wget -q https://archive.apache.org/dist/spark/spark-${APACHE_SPARK_VERSION}/spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME}.tgz && \
     tar xzf spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME}.tgz -C /usr/local && \
-    rm spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME}.tgz
-
-# Overwrite symlink
-RUN cd /usr/local && \
-    rm spark && \
-    ln -s spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME} spark
+    rm spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME}.tgz && \
+    ln -snf /usr/local/spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME} /usr/local/spark
 
 # Remove other scala kernels
 RUN cd /opt/conda/share/jupyter/kernels/ && \

Makefile

@@ -49,7 +49,7 @@ endef
 
 RUN=$(RUN_PREFIX)$(1)$(RUN_SUFFIX)
 
-ENV_OPTS:=APACHE_SPARK_VERSION=$(APACHE_SPARK_VERSION) VERSION=$(VERSION) IS_SNAPSHOT=$(IS_SNAPSHOT)
+ENV_OPTS:=APACHE_SPARK_VERSION=$(APACHE_SPARK_VERSION) SCALA_VERSION=$(SCALA_VERSION) VERSION=$(VERSION) IS_SNAPSHOT=$(IS_SNAPSHOT)
 
 ASSEMBLY_JAR:=toree-assembly-$(VERSION)$(SNAPSHOT).jar
 
@@ -83,7 +83,10 @@ clean: clean-dist
 	@-docker rmi -f $(TOREE_DEV_IMAGE)
 
 .toree-dev-image:
-	@docker build -t $(TOREE_DEV_IMAGE) -f Dockerfile.toree-dev .
+	@docker build -t $(TOREE_DEV_IMAGE) -f Dockerfile.toree-dev \
+		--build-arg APACHE_SPARK_VERSION=$(APACHE_SPARK_VERSION) \
+		--build-arg SCALA_VERSION=$(SCALA_VERSION) \
+		.
 	touch $@
 
 .clean-binder-image:

build.sbt

@@ -18,13 +18,20 @@
 import scala.util.Properties
 import sbtassembly.AssemblyOption
 
+lazy val scala212 = "2.12.17"
+lazy val scala213 = "2.13.8"
+lazy val defaultScalaVersion = sys.env.get("SCALA_VERSION") match {
+  case Some("2.13") => scala213
+  case _ => scala212
+}
+
 // Version settings
 ThisBuild / version := Properties.envOrElse("VERSION", "0.0.0-dev") +
   (if ((ThisBuild / isSnapshot ).value) "-SNAPSHOT" else "")
 ThisBuild / isSnapshot := Properties.envOrElse("IS_SNAPSHOT","true").toBoolean
 ThisBuild / organization := "org.apache.toree.kernel"
-ThisBuild / crossScalaVersions := Seq("2.12.17")
-ThisBuild / scalaVersion := (ThisBuild / crossScalaVersions ).value.head
+ThisBuild / crossScalaVersions := Seq(scala212, scala213)
+ThisBuild / scalaVersion := defaultScalaVersion
 ThisBuild / Dependencies.sparkVersion := {
   val envVar = "APACHE_SPARK_VERSION"
   val defaultVersion = "3.4.4"
@@ -44,7 +51,6 @@ ThisBuild / scalacOptions ++= Seq(
   "-deprecation",
   "-unchecked",
   "-feature",
-  "-Xfatal-warnings",
   "-language:reflectiveCalls",
   "-target:jvm-1.8"
 )
@@ -118,7 +124,7 @@ ThisBuild / credentials += Credentials(Path.userHome / ".ivy2" / ".credentials")
 
 /** Root Toree project. */
 lazy val root = (project in file("."))
-  .settings(name := "toree", crossScalaVersions := Nil)
+  .settings(name := "toree")
   .aggregate(
     macros,protocol,plugins,communication,kernelApi,client,scalaInterpreter,sqlInterpreter,kernel
   )
@@ -203,10 +209,7 @@ lazy val kernel = (project in file("kernel"))
 enablePlugins(ScalaUnidocPlugin)
 (ScalaUnidoc / unidoc / scalacOptions) ++= Seq(
   "-Ymacro-expand:none",
-  "-skip-packages", Seq(
-    "org.apache.pekko",
-    "scala"
-  ).mkString(":"),
+  "-skip-packages", "org.apache.pekko:scala",
   "-no-link-warnings" // Suppresses problems with Scaladoc @throws links
 )
 

client/src/main/scala/org/apache/toree/kernel/protocol/v5/client/Utilities.scala

@@ -26,6 +26,7 @@ import org.apache.toree.kernel.protocol.v5.content.ExecuteRequest
 import org.apache.toree.utils.LogLike
 import play.api.libs.json.{JsPath, Json, JsonValidationError, Reads}
 
+import scala.collection.mutable
 import scala.concurrent.duration._
 
 object Utilities extends LogLike {
@@ -64,7 +65,7 @@ object Utilities extends LogLike {
     val header = Json.parse(message.frames(delimiterIndex + 2)).as[Header]
     val parentHeader = Json.parse(message.frames(delimiterIndex + 3)).validate[ParentHeader].fold[ParentHeader](
       // TODO: Investigate better solution than setting parentHeader to null for {}
-      (invalid: Seq[(JsPath, Seq[JsonValidationError])]) => null, //HeaderBuilder.empty,
+      (invalid: collection.Seq[(JsPath, collection.Seq[JsonValidationError])]) => null, //HeaderBuilder.empty,
       (valid: ParentHeader) => valid
     )
     val metadata = Json.parse(message.frames(delimiterIndex + 4)).as[Metadata]
@@ -78,20 +79,20 @@ object Utilities extends LogLike {
   }
 
   implicit def KernelMessageToZMQMessage(kernelMessage : KernelMessage) : ZMQMessage = {
-    val frames: scala.collection.mutable.ListBuffer[ByteString] = scala.collection.mutable.ListBuffer()
-    kernelMessage.ids.map((id : Array[Byte]) => frames += ByteString.apply(id) )
+    val frames: mutable.ListBuffer[ByteString] = mutable.ListBuffer()
+    kernelMessage.ids.map((id: Array[Byte]) => frames += ByteString.apply(id))
     frames += "<IDS|MSG>"
     frames += kernelMessage.signature
     frames += Json.toJson(kernelMessage.header).toString()
     frames += Json.toJson(kernelMessage.parentHeader).toString()
     frames += Json.toJson(kernelMessage.metadata).toString
     frames += kernelMessage.contentString
-    ZMQMessage(frames  : _*)
+    ZMQMessage(frames.toSeq : _*)
   }
 
   def parseAndHandle[T](json: String, reads: Reads[T], handler: T => Unit) : Unit = {
     Json.parse(json).validate[T](reads).fold(
-      (invalid: Seq[(JsPath, Seq[JsonValidationError])]) =>
+      (invalid: collection.Seq[(JsPath, collection.Seq[JsonValidationError])]) =>
         logger.error(s"Could not parse JSON, ${json}"),
       (content: T) => handler(content)
     )

client/src/main/scala/org/apache/toree/kernel/protocol/v5/client/socket/HeartbeatClient.scala

@@ -60,7 +60,7 @@ class HeartbeatClient(
     case HeartbeatMessage =>
       import scala.concurrent.ExecutionContext.Implicits.global
       val id = java.util.UUID.randomUUID().toString
-      futureMap += (id -> sender)
+      futureMap += (id -> sender())
       logger.info(s"Heartbeat client send: $id")
       val future = socket ? ZMQMessage(ByteString(id.getBytes))
       future.onComplete {

client/src/main/scala/org/apache/toree/kernel/protocol/v5/client/socket/IOPubClient.scala

@@ -76,7 +76,7 @@ class IOPubClient(
     } else {
       logger.warn("Received message with null parent header.")
       logger.debug(s"Kernel message is: $kernelMessage")
-      sender.forward(Failure(new RuntimeException(PARENT_HEADER_NULL_MESSAGE)))
+      sender().forward(Failure(new RuntimeException(PARENT_HEADER_NULL_MESSAGE)))
     }
   }
 

client/src/test/scala/org/apache/toree/kernel/protocol/v5/client/socket/IOPubClientSpec.scala

@@ -36,7 +36,7 @@ import org.mockito.ArgumentMatchers.{eq => mockEq, _}
 import org.mockito.Mockito._
 import org.scalatest.concurrent.{Eventually, ScalaFutures}
 import org.scalatestplus.mockito.MockitoSugar
-import org.scalatest.time.{Milliseconds, Span}
+import org.scalatest.time.{Milliseconds, Seconds, Span}
 import org.scalatest.funspec.AnyFunSpecLike
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.BeforeAndAfterEach
@@ -60,7 +60,7 @@ class IOPubClientSpec extends TestKit(ActorSystem(
 {
   private val TestTimeout = Timeout(10.seconds)
   implicit override val patienceConfig = PatienceConfig(
-    timeout = scaled(Span(200, Milliseconds)),
+    timeout = scaled(Span(1, Seconds)),
     interval = scaled(Span(5, Milliseconds))
   )
   private val SignatureEnabled = true

communication/src/main/scala/org/apache/toree/communication/security/SignatureCheckerActor.scala

@@ -35,7 +35,7 @@ class SignatureCheckerActor(
       val isValidSignature = hmacString == signature
       logger.trace(s"Signature ${signature} validity checked against " +
         s"hmac ${hmacString} with outcome ${isValidSignature}")
-      sender ! isValidSignature
+      sender() ! isValidSignature
     }
   }