From ae9c4bbaeaf41f6c2e56c5a2411f1aa79e28646f Mon Sep 17 00:00:00 2001 From: Martin Mauch Date: Mon, 1 Mar 2021 13:06:36 +0100 Subject: [PATCH 1/3] Add support for running Scala via Almond --- repo2docker/buildpacks/__init__.py | 1 + repo2docker/buildpacks/scala/__init__.py | 1 + repo2docker/buildpacks/scala/scala_project.py | 145 ++++++++++++++++++ tests/scala/scala-2.12.12/build.sbt | 1 + 4 files changed, 148 insertions(+) create mode 100644 repo2docker/buildpacks/scala/__init__.py create mode 100644 repo2docker/buildpacks/scala/scala_project.py create mode 100644 tests/scala/scala-2.12.12/build.sbt diff --git a/repo2docker/buildpacks/__init__.py b/repo2docker/buildpacks/__init__.py index 10fe89a4e..134a7fe97 100644 --- a/repo2docker/buildpacks/__init__.py +++ b/repo2docker/buildpacks/__init__.py @@ -4,6 +4,7 @@ from .conda import CondaBuildPack from .julia import JuliaProjectTomlBuildPack from .julia import JuliaRequireBuildPack +from .scala import ScalaBuildPack from .docker import DockerBuildPack from .legacy import LegacyBinderDockerBuildPack from .r import RBuildPack diff --git a/repo2docker/buildpacks/scala/__init__.py b/repo2docker/buildpacks/scala/__init__.py new file mode 100644 index 000000000..d167a4161 --- /dev/null +++ b/repo2docker/buildpacks/scala/__init__.py @@ -0,0 +1 @@ +from .scala_project import ScalaBuildPack diff --git a/repo2docker/buildpacks/scala/scala_project.py b/repo2docker/buildpacks/scala/scala_project.py new file mode 100644 index 000000000..d4fcb9049 --- /dev/null +++ b/repo2docker/buildpacks/scala/scala_project.py @@ -0,0 +1,145 @@ +"""Generates a Dockerfile based on an input matrix for Scala""" +import os +import toml +from ..python import PythonBuildPack +from .semver import find_semver_match + + +class ScalaBuildPack(PythonBuildPack): + """ + Scala build pack which uses conda. + """ + + # ALL EXISTING SCALA VERSIONS + # Note that these must remain ordered, in order for the find_semver_match() + # function to behave correctly. + all_scalas = [ + "2.12.12", + "2.12.13", + "2.13.4", + "2.13.5" + ] + + @property + def scala_version(self): + default_scala_version = self.all_scalas[-1] + return default_scala_version + + def get_build_env(self): + """Get additional environment settings for Scala and Jupyter + + Returns: + an ordered list of environment setting tuples + + The tuples contain a string of the environment variable name and + a string of the environment setting: + - `SCALA_PATH`: base path where all Scala Binaries and libraries + will be installed + - `SCALA_DEPOT_PATH`: path where Scala libraries are installed. + - `SCALA_VERSION`: default version of scala to be installed + - `JUPYTER`: environment variable required by IScala to point to + the `jupyter` executable + + For example, a tuple may be `('SCALA_VERSION', '2.13.5')`. + + """ + return super().get_build_env() + [ + ("SCALA_PATH", "${APP_BASE}/scala"), + ("SCALA_BIN", "${SCALA_PATH}/bin"), + ("COURSIER_CACHE", "${SCALA_PATH}/pkg"), + ("COURSIER_BIN_DIR", "${SCALA_BIN}"), + ("SCALA_VERSION", self.scala_version), + ("JUPYTER", "${NB_PYTHON_PREFIX}/bin/jupyter"), + ("JUPYTER_DATA_DIR", "${NB_PYTHON_PREFIX}/share/jupyter"), + ] + + def get_env(self): + return super().get_env() + [("SCALA_PROJECT", "${REPO_DIR}")] + + def get_path(self): + """Adds path to Scala binaries to user's PATH. + + Returns: + an ordered list of path strings. The path to the Scala + executable is added to the list. + + """ + return super().get_path() + ["${SCALA_BIN}"] + + def get_build_scripts(self): + """ + Return series of build-steps common to "ALL" Scala repositories + + All scripts found here should be independent of contents of a + particular repository. + + This creates a directory with permissions for installing scala packages + (from get_assemble_scripts). + + """ + return super().get_build_scripts() + [ + ( + "root", + r""" + mkdir -p ${SCALA_BIN} && \ + + apt-get -y update && \ + apt-get install --no-install-recommends -y \ + curl \ + openjdk-8-jre-headless \ + ca-certificates-java && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ + + curl -Lo ${SCALA_BIN}/coursier https://github.com/coursier/coursier/releases/download/v2.0.12/coursier && \ + chmod +x ${SCALA_BIN}/coursier + """, + ), + ( + "root", + r""" + mkdir -p ${COURSIER_CACHE} && \ + chown ${NB_USER}:${NB_USER} ${COURSIER_CACHE} + """, + ), + ] + + def get_assemble_scripts(self): + """ + Return series of build-steps specific to "this" Scala repository + + We make sure that the IScala package gets installed into the default + environment, and not the project specific one, by running the + IScala install command with SCALA_PROJECT="". + + Instantiate and then precompile all packages in the repos scala + environment. + + The parent, CondaBuildPack, will add the build steps for + any needed Python packages found in environment.yml. + """ + return super().get_assemble_scripts() + [ + ( + "${NB_USER}", + r""" + SCALA_PROJECT="" scala -e "using Pkg; Pkg.add(\"IScala\"); using IScala; installkernel(\"Scala\", \"--project=${REPO_DIR}\");" && \ + scala --project=${REPO_DIR} -e 'using Pkg; Pkg.instantiate(); Pkg.resolve(); pkg"precompile"' + """, + ) + ] + + def detect(self): + """ + Check if current repo should be built with the Scala Build pack + + super().detect() is not called in this function - it would return + false unless an `environment.yml` is present and we do not want to + require the presence of a `environment.yml` to use Scala. + + Instead we just check if the path to `Project.toml` or + `ScalaProject.toml` exists. + + """ + return os.path.exists(self.binder_path("build.sc")) or os.path.exists( + self.binder_path("build.sbt") + ) diff --git a/tests/scala/scala-2.12.12/build.sbt b/tests/scala/scala-2.12.12/build.sbt new file mode 100644 index 000000000..366c27de7 --- /dev/null +++ b/tests/scala/scala-2.12.12/build.sbt @@ -0,0 +1 @@ +scalaVersion := "2.12.12" From d1f7523610af332fdf5c4f8de79da935ef0f71d9 Mon Sep 17 00:00:00 2001 From: Martin Mauch Date: Mon, 1 Mar 2021 15:45:25 +0100 Subject: [PATCH 2/3] fixup! Add support for running Scala via Almond --- repo2docker/app.py | 2 ++ repo2docker/buildpacks/scala/scala_project.py | 22 +++++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/repo2docker/app.py b/repo2docker/app.py index 8a804905c..81fe3b8dd 100755 --- a/repo2docker/app.py +++ b/repo2docker/app.py @@ -32,6 +32,7 @@ DockerBuildPack, JuliaProjectTomlBuildPack, JuliaRequireBuildPack, + ScalaBuildPack, LegacyBinderDockerBuildPack, NixBuildPack, PipfileBuildPack, @@ -94,6 +95,7 @@ def _default_log_level(self): DockerBuildPack, JuliaProjectTomlBuildPack, JuliaRequireBuildPack, + ScalaBuildPack, NixBuildPack, RBuildPack, CondaBuildPack, diff --git a/repo2docker/buildpacks/scala/scala_project.py b/repo2docker/buildpacks/scala/scala_project.py index d4fcb9049..d71761490 100644 --- a/repo2docker/buildpacks/scala/scala_project.py +++ b/repo2docker/buildpacks/scala/scala_project.py @@ -2,8 +2,7 @@ import os import toml from ..python import PythonBuildPack -from .semver import find_semver_match - +from ..base import BuildPack, BaseImage class ScalaBuildPack(PythonBuildPack): """ @@ -16,8 +15,7 @@ class ScalaBuildPack(PythonBuildPack): all_scalas = [ "2.12.12", "2.12.13", - "2.13.4", - "2.13.5" + "2.13.4" ] @property @@ -48,7 +46,10 @@ def get_build_env(self): ("SCALA_BIN", "${SCALA_PATH}/bin"), ("COURSIER_CACHE", "${SCALA_PATH}/pkg"), ("COURSIER_BIN_DIR", "${SCALA_BIN}"), + ("ALMOND_VERSION", "0.11.0"), ("SCALA_VERSION", self.scala_version), + #("SCALA_VERSION_MAJOR_TRIMMED", self.scala_version.split), + ("SCALA_MAJOR_VERSION_TRIMMED", "2.13"), ("JUPYTER", "${NB_PYTHON_PREFIX}/bin/jupyter"), ("JUPYTER_DATA_DIR", "${NB_PYTHON_PREFIX}/share/jupyter"), ] @@ -92,7 +93,15 @@ def get_build_scripts(self): rm -rf /var/lib/apt/lists/* && \ curl -Lo ${SCALA_BIN}/coursier https://github.com/coursier/coursier/releases/download/v2.0.12/coursier && \ - chmod +x ${SCALA_BIN}/coursier + chmod +x ${SCALA_BIN}/coursier && \ + curl -Lo ${SCALA_BIN}/install-kernels.sh https://raw.githubusercontent.com/almond-sh/almond/master/scripts/install-kernels.sh && \ + coursier bootstrap \ + -r jitpack \ + -i user -I user:sh.almond:scala-kernel-api_${SCALA_VERSION}:${ALMOND_VERSION} \ + sh.almond:scala-kernel_${SCALA_VERSION}:${ALMOND_VERSION} \ + --default=true --sources \ + -o ${SCALA_BIN}/almond && \ + ${SCALA_BIN}/almond --install --log info --metabrowse --id scala${SCALA_MAJOR_VERSION_TRIMMED} --display-name "Scala ${SCALA_MAJOR_VERSION}" --jupyter-path ${NB_PYTHON_PREFIX}/share/jupyter/kernels/ """, ), ( @@ -122,8 +131,7 @@ def get_assemble_scripts(self): ( "${NB_USER}", r""" - SCALA_PROJECT="" scala -e "using Pkg; Pkg.add(\"IScala\"); using IScala; installkernel(\"Scala\", \"--project=${REPO_DIR}\");" && \ - scala --project=${REPO_DIR} -e 'using Pkg; Pkg.instantiate(); Pkg.resolve(); pkg"precompile"' + echo Implement me """, ) ] From 9c9d41e6db9673505a9907752914513941c96923 Mon Sep 17 00:00:00 2001 From: Martin Mauch Date: Tue, 2 Mar 2021 10:28:46 +0100 Subject: [PATCH 3/3] fixup! Add support for running Scala via Almond --- repo2docker/buildpacks/scala/scala_project.py | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/repo2docker/buildpacks/scala/scala_project.py b/repo2docker/buildpacks/scala/scala_project.py index d71761490..a1ecc3e63 100644 --- a/repo2docker/buildpacks/scala/scala_project.py +++ b/repo2docker/buildpacks/scala/scala_project.py @@ -14,7 +14,6 @@ class ScalaBuildPack(PythonBuildPack): # function to behave correctly. all_scalas = [ "2.12.12", - "2.12.13", "2.13.4" ] @@ -90,27 +89,25 @@ def get_build_scripts(self): openjdk-8-jre-headless \ ca-certificates-java && \ apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - - curl -Lo ${SCALA_BIN}/coursier https://github.com/coursier/coursier/releases/download/v2.0.12/coursier && \ - chmod +x ${SCALA_BIN}/coursier && \ - curl -Lo ${SCALA_BIN}/install-kernels.sh https://raw.githubusercontent.com/almond-sh/almond/master/scripts/install-kernels.sh && \ - coursier bootstrap \ - -r jitpack \ - -i user -I user:sh.almond:scala-kernel-api_${SCALA_VERSION}:${ALMOND_VERSION} \ - sh.almond:scala-kernel_${SCALA_VERSION}:${ALMOND_VERSION} \ - --default=true --sources \ - -o ${SCALA_BIN}/almond && \ - ${SCALA_BIN}/almond --install --log info --metabrowse --id scala${SCALA_MAJOR_VERSION_TRIMMED} --display-name "Scala ${SCALA_MAJOR_VERSION}" --jupyter-path ${NB_PYTHON_PREFIX}/share/jupyter/kernels/ - """, + rm -rf /var/lib/apt/lists/* + """ ), ( "root", r""" + curl -Lo ${SCALA_BIN}/coursier https://github.com/coursier/coursier/releases/download/v2.0.12/coursier && \ + chmod +x ${SCALA_BIN}/coursier && \ mkdir -p ${COURSIER_CACHE} && \ chown ${NB_USER}:${NB_USER} ${COURSIER_CACHE} - """, + """ ), + ( + "${NB_USER}", + r""" + coursier launch almond --scala 2.13.4 -- --install --log info --metabrowse --id scala213 --display-name "Scala 2.13" --jupyter-path ${NB_PYTHON_PREFIX}/share/jupyter/kernels/ && \ + coursier launch almond --scala 2.12.12 -- --install --log info --metabrowse --id scala212 --display-name "Scala 2.12" --jupyter-path ${NB_PYTHON_PREFIX}/share/jupyter/kernels/ + """, + ) ] def get_assemble_scripts(self):