Skip to content

Commit 1c95627

Browse files
pan3793pan3793
committed
[TOREE-557] Bump Spark 3.5
1 parent 92ce603 commit 1c95627

File tree

6 files changed

+50
-44
lines changed

6 files changed

+50
-44
lines changed

‎Dockerfile‎

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -26,32 +26,32 @@ RUN curl -sL https://deb.nodesource.com/setup_0.12 | bash - && \
2626
npm install -g bower
2727

2828
# for Apache Spark demos
29-
ARG APACHE_SPARK_VERSION=3.3.2
29+
ARG APACHE_SPARK_VERSION=3.5.2
3030
ARG SCALA_VERSION=2.13
3131

3232
RUN apt-get -y update && \
3333
apt-get -y install software-properties-common
3434

35-
RUN \
36-
echo "===> add webupd8 repository..." && \
37-
echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list && \
38-
echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list && \
39-
apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EEA14886 && \
40-
apt-get update
41-
42-
RUN echo "===> install Java" && \
43-
echo debconf shared/accepted-oracle-license-v1-1 select true | debconf-set-selections && \
44-
echo debconf shared/accepted-oracle-license-v1-1 seen true | debconf-set-selections && \
45-
DEBIAN_FRONTEND=noninteractive apt-get install -y --force-yes oracle-java8-installer oracle-java8-set-default && \
46-
apt-get clean && \
47-
update-java-alternatives -s java-8-oracle
48-
49-
RUN cd /tmp && \
50-
if [ "$SCALA_VERSION" = "2.13" ]; then APACHE_SPARK_CUSTOM_NAME=hadoop3-scala2.13; else APACHE_SPARK_CUSTOM_NAME=hadoop3; fi && \
51-
wget -q https://archive.apache.org/dist/spark/spark-${APACHE_SPARK_VERSION}/spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME}.tgz && \
52-
tar xzf spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME}.tgz -C /usr/local && \
53-
rm spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME}.tgz && \
54-
ln -snf /usr/local/spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME} /usr/local/spark
35+
RUN JAVA_8=`update-alternatives --list java | grep java-8-openjdk` || echo $JAVA_8 && \
36+
if [ "x$JAVA_8" = "x" ]; then \
37+
apt-get -y update ; \
38+
apt-get install -y --no-install-recommends openjdk-8-jdk ca-certificates-java ; \
39+
apt-get clean ; \
40+
rm -rf /var/lib/apt/lists/* ; \
41+
update-ca-certificates -f ; \
42+
JAVA_8=`update-alternatives --list java | grep java-8-openjdk` ; \
43+
update-alternatives --set java $JAVA_8 ; \
44+
fi
45+
46+
RUN if [ "$SCALA_VERSION" = "2.13" ]; then APACHE_SPARK_CUSTOM_NAME=hadoop3-scala2.13; else APACHE_SPARK_CUSTOM_NAME=hadoop3; fi && \
47+
SPARK_TGZ_NAME=spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME} && \
48+
if [ ! -d "/usr/local/$SPARK_TGZ_NAME" ]; then \
49+
cd /tmp ; \
50+
wget -q https://archive.apache.org/dist/spark/spark-${APACHE_SPARK_VERSION}/${SPARK_TGZ_NAME}.tgz ; \
51+
tar -xzf ${SPARK_TGZ_NAME}.tgz -C /usr/local ; \
52+
rm ${SPARK_TGZ_NAME}.tgz ; \
53+
ln -snf /usr/local/$SPARK_TGZ_NAME /usr/local/spark ; \
54+
fi
5555

5656
# R support
5757
RUN apt-get update && \
@@ -62,7 +62,7 @@ RUN apt-get update && \
6262
rm -rf /var/lib/apt/lists/*
6363

6464
ENV SPARK_HOME /usr/local/spark
65-
ENV PYTHONPATH $SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9.5-src.zip
65+
ENV PYTHONPATH $SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9.7-src.zip
6666
ENV PYSPARK_PYTHON /home/main/anaconda2/envs/python3/bin/python
6767
ENV R_LIBS_USER $SPARK_HOME/R/lib
6868

‎Dockerfile.toree-dev‎

Lines changed: 20 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -23,24 +23,30 @@ FROM jupyter/all-spark-notebook
2323
USER root
2424

2525
# Spark dependencies
26-
ARG APACHE_SPARK_VERSION=3.3.2
26+
ARG APACHE_SPARK_VERSION=3.5.2
2727
ARG SCALA_VERSION=2.13
2828

29-
RUN apt-get -y update && \
30-
apt-get install -y --no-install-recommends openjdk-8-jdk ca-certificates-java && \
31-
apt-get clean && \
32-
rm -rf /var/lib/apt/lists/* && \
33-
update-ca-certificates -f && \
34-
JAVA_8=`update-alternatives --list java | grep java-8-openjdk` && \
35-
update-alternatives --set java $JAVA_8
29+
RUN JAVA_8=`update-alternatives --list java | grep java-8-openjdk` || echo $JAVA_8 && \
30+
if [ "x$JAVA_8" = "x" ]; then \
31+
apt-get -y update ; \
32+
apt-get install -y --no-install-recommends openjdk-8-jdk ca-certificates-java ; \
33+
apt-get clean ; \
34+
rm -rf /var/lib/apt/lists/* ; \
35+
update-ca-certificates -f ; \
36+
JAVA_8=`update-alternatives --list java | grep java-8-openjdk` ; \
37+
update-alternatives --set java $JAVA_8 ; \
38+
fi
3639

3740
# Installing Spark3
38-
RUN cd /tmp && \
39-
if [ "$SCALA_VERSION" = "2.13" ]; then APACHE_SPARK_CUSTOM_NAME=hadoop3-scala2.13; else APACHE_SPARK_CUSTOM_NAME=hadoop3; fi && \
40-
wget -q https://archive.apache.org/dist/spark/spark-${APACHE_SPARK_VERSION}/spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME}.tgz && \
41-
tar xzf spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME}.tgz -C /usr/local && \
42-
rm spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME}.tgz && \
43-
ln -snf /usr/local/spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME} /usr/local/spark
41+
RUN if [ "$SCALA_VERSION" = "2.13" ]; then APACHE_SPARK_CUSTOM_NAME=hadoop3-scala2.13; else APACHE_SPARK_CUSTOM_NAME=hadoop3; fi && \
42+
SPARK_TGZ_NAME=spark-${APACHE_SPARK_VERSION}-bin-${APACHE_SPARK_CUSTOM_NAME} && \
43+
if [ ! -d "/usr/local/$SPARK_TGZ_NAME" ]; then \
44+
cd /tmp ; \
45+
wget -q https://archive.apache.org/dist/spark/spark-${APACHE_SPARK_VERSION}/${SPARK_TGZ_NAME}.tgz ; \
46+
tar -xzf ${SPARK_TGZ_NAME}.tgz -C /usr/local ; \
47+
rm ${SPARK_TGZ_NAME}.tgz ; \
48+
ln -snf /usr/local/$SPARK_TGZ_NAME /usr/local/spark ; \
49+
fi
4450

4551
# Remove other scala kernels
4652
RUN cd /opt/conda/share/jupyter/kernels/ && \

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ IS_SNAPSHOT?=true
3030
SNAPSHOT:=-SNAPSHOT
3131
endif
3232

33-
APACHE_SPARK_VERSION?=3.3.2
33+
APACHE_SPARK_VERSION?=3.5.2
3434
SCALA_VERSION?=2.13
3535
IMAGE?=jupyter/all-spark-notebook:latest
3636
EXAMPLE_IMAGE?=apache/toree-examples

‎build.sbt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
import scala.util.Properties
1919

20-
lazy val scala212 = "2.12.15"
20+
lazy val scala212 = "2.12.18"
2121
lazy val scala213 = "2.13.8"
2222
lazy val defaultScalaVersion = sys.env.get("SCALA_VERSION") match {
2323
case Some("2.12") => scala212
@@ -33,7 +33,7 @@ ThisBuild / crossScalaVersions := Seq(scala212, scala213)
3333
ThisBuild / scalaVersion := defaultScalaVersion
3434
ThisBuild / Dependencies.sparkVersion := {
3535
val envVar = "APACHE_SPARK_VERSION"
36-
val defaultVersion = "3.3.2"
36+
val defaultVersion = "3.5.2"
3737

3838
Properties.envOrNone(envVar) match {
3939
case None =>

‎etc/kernel.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
},
55
"display_name": "Apache Toree (development)",
66
"env": {
7-
"PYTHONPATH": "/usr/local/spark/python:/usr/local/spark/python/lib/py4j-0.10.9.5-src.zip",
7+
"PYTHONPATH": "/usr/local/spark/python:/usr/local/spark/python/lib/py4j-0.10.9.7-src.zip",
88
"SPARK_HOME": "/usr/local/spark",
99
"CAPTURE_STANDARD_ERR": "true",
1010
"MAX_INTERPRETER_THREADS": "16",

‎project/Dependencies.scala‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,14 +32,14 @@ object Dependencies {
3232

3333
val config = "com.typesafe" % "config" % "1.4.3" // Apache v2
3434

35-
val coursierVersion = "2.0.0"
35+
val coursierVersion = "2.0.16"
3636
val coursier = "io.get-coursier" %% "coursier" % coursierVersion // Apache v2
3737
val coursierCache = "io.get-coursier" %% "coursier-cache" % coursierVersion // Apache v2
3838

3939
val ivy = "org.apache.ivy" % "ivy" % "2.5.1" // Apache v2
4040

41-
// use the same jackson version in test than the one provided at runtime by Spark 3.3.2
42-
val jacksonDatabind = "com.fasterxml.jackson.core" % "jackson-databind" % "2.13.4.2" // Apache v2
41+
// use the same jackson version in test than the one provided at runtime by Spark 3.5.2
42+
val jacksonDatabind = "com.fasterxml.jackson.core" % "jackson-databind" % "2.15.2" // Apache v2
4343

4444
val jeroMq = "org.zeromq" % "jeromq" % "0.5.3" // MPL v2
4545

@@ -55,7 +55,7 @@ object Dependencies {
5555
val scalaTestMockito = "org.scalatestplus" %% "mockito-4-11" % "3.2.16.0" // Apache v2
5656
val mockitoInline = "org.mockito" % "mockito-inline" % "4.11.0" // MIT
5757

58-
val slf4jApi = "org.slf4j" % "slf4j-api" % "1.7.32" // MIT
58+
val slf4jApi = "org.slf4j" % "slf4j-api" % "2.0.7" // MIT
5959

6060
val sparkVersion = settingKey[String]("Version of Apache Spark to use in Toree") // defined in root build
6161
val sparkCore = Def.setting{ "org.apache.spark" %% "spark-core" % sparkVersion.value } // Apache v2

0 commit comments

Comments
 (0)