diff --git a/Dockerfile b/Dockerfile index 9b2eb1a7..5cefb6a7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,32 +26,32 @@ RUN curl -sL https://deb.nodesource.com/setup_0.12 | bash - && \ npm install -g bower # for Apache Spark demos -ENV APACHE_SPARK_VERSION 3.4.4 +ENV APACHE_SPARK_VERSION 3.5.7 ARG SCALA_VERSION=2.12 RUN apt-get -y update && \ apt-get -y install software-properties-common -RUN \ - echo "===> add webupd8 repository..." && \ - echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list && \ - echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list && \ - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EEA14886 && \ - apt-get update - -RUN echo "===> install Java" && \ - echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \ - echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes oracle-java8-installer oracle-java8-set-default && \ - apt-get clean && \ - update-java-alternatives -s java-8-oracle - -RUN cd /tmp && \ - if [ "$SCALA_VERSION" = "2.13" ]; then APACHE_SPARK_CUSTOM_NAME=hadoop3-scala2.13; else APACHE_SPARK_CUSTOM_NAME=hadoop3; fi && \ - wget -q https://archive.apache.org/dist/spark/spark-${APACHE_SPARK_VERSION}/spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME}.tgz && \ - tar xzf spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME}.tgz -C /usr/local && \ - rm spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME}.tgz && \ - ln -snf /usr/local/spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME} /usr/local/spark +RUN JAVA_8=`update-alternatives --list java | grep java-1.8.0-openjdk` || echo $JAVA_8 && \ + if [ "x$JAVA_8" = "x" ]; then \ + apt-get -y update ; \ + apt-get install -y --no-install-recommends openjdk-8-jdk ca-certificates-java ; \ + apt-get clean ; \ + rm -rf /var/lib/apt/lists/* ; \ + update-ca-certificates -f ; \ + JAVA_8=`update-java-alternatives --list | grep java-1.8.0-openjdk | awk '{print $NF}'` ; \ + update-java-alternatives --set $JAVA_8 ; \ + fi + +RUN if [ "$SCALA_VERSION" = "2.13" ]; then APACHE_SPARK_CUSTOM_NAME=hadoop3-scala2.13; else APACHE_SPARK_CUSTOM_NAME=hadoop3; fi && \ + SPARK_TGZ_NAME=spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME} && \ + if [ ! -d "/usr/local/$SPARK_TGZ_NAME" ]; then \ + cd /tmp ; \ + wget -q https://www.apache.org/dyn/closer.lua/spark/spark-${APACHE_SPARK_VERSION}/${SPARK_TGZ_NAME}.tgz?action=download -O ${SPARK_TGZ_NAME}.tgz ; \ + tar -xzf ${SPARK_TGZ_NAME}.tgz -C /usr/local ; \ + rm ${SPARK_TGZ_NAME}.tgz ; \ + ln -snf /usr/local/$SPARK_TGZ_NAME /usr/local/spark ; \ + fi # R support RUN apt-get update && \ @@ -62,7 +62,7 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* ENV SPARK_HOME /usr/local/spark -ENV PYTHONPATH $SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9.5-src.zip +ENV PYTHONPATH $SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9.7-src.zip ENV PYSPARK_PYTHON /home/main/anaconda2/envs/python3/bin/python ENV R_LIBS_USER $SPARK_HOME/R/lib diff --git a/Dockerfile.toree-dev b/Dockerfile.toree-dev index 731b9c83..f3447fd1 100644 --- a/Dockerfile.toree-dev +++ b/Dockerfile.toree-dev @@ -23,24 +23,30 @@ FROM jupyter/all-spark-notebook USER root # Spark dependencies -ARG APACHE_SPARK_VERSION=3.4.4 +ARG APACHE_SPARK_VERSION=3.5.7 ARG SCALA_VERSION=2.12 -RUN apt-get -y update && \ - apt-get install -y --no-install-recommends openjdk-8-jdk ca-certificates-java && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - update-ca-certificates -f && \ - JAVA_8=`update-alternatives --list java | grep java-8-openjdk` && \ - update-alternatives --set java $JAVA_8 +RUN JAVA_8=`update-alternatives --list java | grep java-1.8.0-openjdk` || echo $JAVA_8 && \ + if [ "x$JAVA_8" = "x" ]; then \ + apt-get -y update ; \ + apt-get install -y --no-install-recommends openjdk-8-jdk ca-certificates-java ; \ + apt-get clean ; \ + rm -rf /var/lib/apt/lists/* ; \ + update-ca-certificates -f ; \ + JAVA_8=`update-java-alternatives --list | grep java-1.8.0-openjdk | awk '{print $NF}'` ; \ + update-java-alternatives --set $JAVA_8 ; \ + fi # Installing Spark3 -RUN cd /tmp && \ - if [ "$SCALA_VERSION" = "2.13" ]; then APACHE_SPARK_CUSTOM_NAME=hadoop3-scala2.13; else APACHE_SPARK_CUSTOM_NAME=hadoop3; fi && \ - wget -q https://archive.apache.org/dist/spark/spark-${APACHE_SPARK_VERSION}/spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME}.tgz && \ - tar xzf spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME}.tgz -C /usr/local && \ - rm spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME}.tgz && \ - ln -snf /usr/local/spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME} /usr/local/spark +RUN if [ "$SCALA_VERSION" = "2.13" ]; then APACHE_SPARK_CUSTOM_NAME=hadoop3-scala2.13; else APACHE_SPARK_CUSTOM_NAME=hadoop3; fi && \ + SPARK_TGZ_NAME=spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME} && \ + if [ ! -d "/usr/local/$SPARK_TGZ_NAME" ]; then \ + cd /tmp ; \ + wget -q https://www.apache.org/dyn/closer.lua/spark/spark-${APACHE_SPARK_VERSION}/${SPARK_TGZ_NAME}.tgz?action=download -O ${SPARK_TGZ_NAME}.tgz ; \ + tar -xzf ${SPARK_TGZ_NAME}.tgz -C /usr/local ; \ + rm ${SPARK_TGZ_NAME}.tgz ; \ + ln -snf /usr/local/$SPARK_TGZ_NAME /usr/local/spark ; \ + fi # Remove other scala kernels RUN cd /opt/conda/share/jupyter/kernels/ && \ diff --git a/Makefile b/Makefile index a2bc3117..a7d20429 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ IS_SNAPSHOT?=true SNAPSHOT:=-SNAPSHOT endif -APACHE_SPARK_VERSION?=3.4.4 +APACHE_SPARK_VERSION?=3.5.7 SCALA_VERSION?=2.12 IMAGE?=jupyter/all-spark-notebook:latest EXAMPLE_IMAGE?=apache/toree-examples diff --git a/build.sbt b/build.sbt index 3fb4030b..ab981c8f 100644 --- a/build.sbt +++ b/build.sbt @@ -18,7 +18,7 @@ import scala.util.Properties import sbtassembly.AssemblyOption -lazy val scala212 = "2.12.17" +lazy val scala212 = "2.12.18" lazy val scala213 = "2.13.8" lazy val defaultScalaVersion = sys.env.get("SCALA_VERSION") match { case Some("2.13") => scala213 @@ -34,7 +34,7 @@ ThisBuild / crossScalaVersions := Seq(scala212, scala213) ThisBuild / scalaVersion := defaultScalaVersion ThisBuild / Dependencies.sparkVersion := { val envVar = "APACHE_SPARK_VERSION" - val defaultVersion = "3.4.4" + val defaultVersion = "3.5.7" Properties.envOrNone(envVar) match { case None => diff --git a/etc/kernel.json b/etc/kernel.json index 868215ac..98808659 100644 --- a/etc/kernel.json +++ b/etc/kernel.json @@ -4,7 +4,7 @@ }, "display_name": "Apache Toree (development)", "env": { - "PYTHONPATH": "/usr/local/spark/python:/usr/local/spark/python/lib/py4j-0.10.9.5-src.zip", + "PYTHONPATH": "/usr/local/spark/python:/usr/local/spark/python/lib/py4j-0.10.9.7-src.zip", "SPARK_HOME": "/usr/local/spark", "CAPTURE_STANDARD_ERR": "true", "MAX_INTERPRETER_THREADS": "16", diff --git a/plugins/build.sbt b/plugins/build.sbt index 88f9a494..1eaef52d 100644 --- a/plugins/build.sbt +++ b/plugins/build.sbt @@ -20,6 +20,9 @@ Test / fork := true // Needed for type inspection libraryDependencies ++= Seq( Dependencies.scalaReflect.value, + Dependencies.asm, + Dependencies.asmCommons, + Dependencies.asmUtil, Dependencies.clapper, Dependencies.slf4jApi ) diff --git a/plugins/src/main/scala/org/apache/toree/plugins/PluginSearcher.scala b/plugins/src/main/scala/org/apache/toree/plugins/PluginSearcher.scala index 75fa6f92..5e2a661e 100644 --- a/plugins/src/main/scala/org/apache/toree/plugins/PluginSearcher.scala +++ b/plugins/src/main/scala/org/apache/toree/plugins/PluginSearcher.scala @@ -17,7 +17,8 @@ package org.apache.toree.plugins import java.io.File -import org.clapper.classutil.{ClassInfo, ClassFinder} +import org.clapper.classutil.{ClassFinder, ClassInfo} +import org.objectweb.asm.Opcodes import org.slf4j.LoggerFactory import scala.annotation.tailrec @@ -52,7 +53,7 @@ class PluginSearcher { * * @return The new class finder */ - protected def newClassFinder(): ClassFinder = ClassFinder(classpath) + protected def newClassFinder(): ClassFinder = ClassFinder(classpath, Some(Opcodes.ASM9)) /** * Creates a new class finder for the given paths. @@ -61,7 +62,7 @@ class PluginSearcher { * * @return The new class finder */ - protected def newClassFinder(paths: Seq[File]): ClassFinder = ClassFinder(paths) + protected def newClassFinder(paths: Seq[File]): ClassFinder = ClassFinder(paths, Some(Opcodes.ASM9)) /** * Loads all class information using the provided class finder. diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 08d7bf86..3db4af81 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -26,20 +26,24 @@ object Dependencies { val pekkoSlf4j = "org.apache.pekko" %% "pekko-slf4j" % pekkoVersion // Apache v2 val pekkoTestkit = "org.apache.pekko" %% "pekko-testkit" % pekkoVersion // Apache v2 - val clapper = "org.clapper" %% "classutil" % "1.5.1" // BSD 3-clause license, used for detecting plugins + val asmVersion = "9.9" + val asm = "org.ow2.asm" % "asm" % asmVersion // Apache v2 + val asmCommons = "org.ow2.asm" % "asm-commons" % asmVersion // Apache v2 + val asmUtil = "org.ow2.asm" % "asm-util" % asmVersion // Apache v2 + val clapper = "org.clapper" %% "classutil" % "1.5.1" // Apache v2, used for detecting plugins val commonsExec = "org.apache.commons" % "commons-exec" % "1.3" // Apache v2 val config = "com.typesafe" % "config" % "1.4.3" // Apache v2 - val coursierVersion = "2.0.0" + val coursierVersion = "2.0.16" val coursier = "io.get-coursier" %% "coursier" % coursierVersion // Apache v2 val coursierCache = "io.get-coursier" %% "coursier-cache" % coursierVersion // Apache v2 val ivy = "org.apache.ivy" % "ivy" % "2.5.1" // Apache v2 - // use the same jackson version in test than the one provided at runtime by Spark 3.4.x - val jacksonDatabind = "com.fasterxml.jackson.core" % "jackson-databind" % "2.14.2" // Apache v2 + // use the same jackson version in test than the one provided at runtime by Spark 3.5.x + val jacksonDatabind = "com.fasterxml.jackson.core" % "jackson-databind" % "2.15.2" // Apache v2 val jeroMq = "org.zeromq" % "jeromq" % "0.5.3" // MPL v2 @@ -57,7 +61,7 @@ object Dependencies { val scalaTestMockito = "org.scalatestplus" %% "mockito-4-11" % "3.2.16.0" // Apache v2 val mockitoInline = "org.mockito" % "mockito-inline" % "4.11.0" // MIT - val slf4jApi = "org.slf4j" % "slf4j-api" % "2.0.6" // MIT + val slf4jApi = "org.slf4j" % "slf4j-api" % "2.0.7" // MIT val sparkVersion = settingKey[String]("Version of Apache Spark to use in Toree") // defined in root build val sparkCore = Def.setting{ "org.apache.spark" %% "spark-core" % sparkVersion.value } // Apache v2