Skip to content

Commit 889d0f3

Browse files
lukasmasuchlukasmasuch
committed
Update and improve tool scripts
1 parent d862dc9 commit 889d0f3

File tree

13 files changed

+88
-28
lines changed

13 files changed

+88
-28
lines changed

‎resources/tools/filebrowser.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ if [ ! -f "/usr/local/bin/filebrowser" ]; then
1818
echo "Installing Filebrowser. Please wait..."
1919
mkdir -p $RESOURCES_PATH/filebrowser
2020
cd $RESOURCES_PATH/filebrowser
21-
wget -q https://github.com/filebrowser/filebrowser/releases/download/v2.10.0/linux-amd64-filebrowser.tar.gz -O ./filebrowser.tar.gz
21+
FILEBROWSER_VERSION=2.11.0
22+
wget -q https://github.com/filebrowser/filebrowser/releases/download/v$FILEBROWSER_VERSION/linux-amd64-filebrowser.tar.gz -O ./filebrowser.tar.gz
2223
tar -xzf ./filebrowser.tar.gz
2324
chmod +x "./filebrowser"
2425
mv "./filebrowser" "/usr/local/bin/filebrowser"

‎resources/tools/flink-local-cluster.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ if [ ! -d "$FLINK_HOME" ]; then
2222
cd $RESOURCES_PATH
2323
FLINK_VERSION=1.12.0
2424
SCALA_VERSION=2.12
25-
wget https://ftp.fau.de/apache/flink/flink-$FLINK_VERSION/flink-$FLINK_VERSION-bin-scala_$SCALA_VERSION.tgz -O ./flink.tar.gz
25+
echo "Downloading. Please wait..."
26+
wget -q https://ftp.fau.de/apache/flink/flink-$FLINK_VERSION/flink-$FLINK_VERSION-bin-scala_$SCALA_VERSION.tgz -O ./flink.tar.gz
2627
tar xzf flink.tar.gz
2728
mv flink-$FLINK_VERSION $FLINK_HOME
2829
rm flink.tar.gz

‎resources/tools/gpuview.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ done
1717
if ! hash gpuview 2>/dev/null; then
1818
echo "Installing GPUview. Please wait..."
1919
# https://github.com/fgaim/gpuview
20-
pip install --no-cache-dir gpuview
20+
pipx install gpuview
2121
else
2222
echo "GPUview is already installed"
2323
fi

‎resources/tools/hadoop-local-cluster.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@ if [ ! -d "$HADOOP_HOME" ]; then
3434
echo "Installing Hadoop. Please wait..."
3535
cd $RESOURCES_PATH
3636
HADOOP_VERSION=3.3.0
37-
wget https://apache.mirror.digionline.de/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz -O ./hadoop.tar.gz
37+
echo "Downloading..."
38+
wget -q https://apache.mirror.digionline.de/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz -O ./hadoop.tar.gz
3839
tar xzf hadoop.tar.gz
3940
mv hadoop-$HADOOP_VERSION $HADOOP_HOME
4041
rm hadoop.tar.gz

‎resources/tools/label-studio.sh‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/bin/sh
2+
3+
# Stops script execution if a command has an error
4+
set -e
5+
6+
INSTALL_ONLY=0
7+
PORT=""
8+
# Loop through arguments and process them: https://pretzelhands.com/posts/command-line-flags
9+
for arg in "$@"; do
10+
case $arg in
11+
-i|--install) INSTALL_ONLY=1 ; shift ;;
12+
-p=*|--port=*) PORT="${arg#*=}" ; shift ;; # TODO Does not allow --port 1234
13+
*) break ;;
14+
esac
15+
done
16+
17+
if ! hash label-studio 2>/dev/null; then
18+
echo "Installing Label Studio. Please wait..."
19+
pipx install label-studio
20+
else
21+
echo "Label Studio is already installed"
22+
fi
23+
24+
# Run
25+
if [ $INSTALL_ONLY = 0 ] ; then
26+
if [ -z "$PORT" ]; then
27+
read -p "Please provide a port for starting Label Studio: " PORT
28+
fi
29+
30+
echo "Starting Label Studio on port "$PORT
31+
cd $WORKSPACE_HOME
32+
# Create tool entry for tooling plugin
33+
echo '{"id": "label-studio-link", "name": "Label Studio", "url_path": "/tools/'$PORT'/import", "description": "Multi-type data labeling & annotation tool"}' > $HOME/.workspace/tools/label-studio.json
34+
label-studio start labeling_project --allow-serving-local-files --init -p $PORT --no-browser
35+
sleep 15
36+
fi

‎resources/tools/netron.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ done
1717
if ! hash netron 2>/dev/null; then
1818
echo "Installing Netron. Please wait..."
1919
# https://github.com/lutzroeder/netron
20-
pip install --no-cache-dir netron
20+
pipx install netron
2121
else
2222
echo "Netron is already installed"
2323
fi

‎resources/tools/oh-my-zsh.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ if ! hash zsh 2>/dev/null; then
2727
rm -rf fonts
2828
# Install plugins
2929
apt-get install -y --no-install-recommends autojump git-flow git-extras ncdu htop
30-
pip install Pygments ranger-fm thefuck
30+
pip install Pygments ranger-fm thefuck bpytop
31+
# Install fkill-cli: (too big - 30MB) npm install --global fkill-cli && \
3132
yes | sh -c "$(curl -fsSL https://raw.github.com/robbyrussell/oh-my-zsh/master/tools/install.sh)"
3233
# Install powerlevel10k for instant prompt
3334
# git clone --depth=1 https://github.com/romkatv/powerlevel10k.git ${ZSH_CUSTOM:-$HOME/.oh-my-zsh/custom}/themes/powerlevel10k

‎resources/tools/portainer.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ done
1717
if [ ! -f "$RESOURCES_PATH/portainer/portainer" ]; then
1818
echo "Installing Portainer. Please wait..."
1919
cd $RESOURCES_PATH
20-
PORTAINER_VERSION=2.0.0
20+
PORTAINER_VERSION=2.0.1
2121
wget https://github.com/portainer/portainer/releases/download/$PORTAINER_VERSION/portainer-$PORTAINER_VERSION-linux-amd64.tar.gz
2222
tar xvpfz portainer-$PORTAINER_VERSION-linux-amd64.tar.gz
2323
rm ./portainer-$PORTAINER_VERSION-linux-amd64.tar.gz

‎resources/tools/scala-utils.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ else
4040
fi
4141

4242

43+
# TODO: Install Coursier https://get-coursier.io/docs/cli-installation
44+
4345
if [ ! -d "$HOME/.local/share/jupyter/kernels/scala" ]; then
4446
echo "Installing Almond Scala Kernel for Jupyter. Please wait..."
4547
curl -Lo coursier https://git.io/coursier-cli

‎resources/tools/spark-local-cluster.sh‎

Lines changed: 36 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -15,18 +15,33 @@ done
1515
# Script inspired by: https://github.com/jupyter/docker-stacks/blob/master/pyspark-notebook/Dockerfile#L18
1616
# https://github.com/apache/incubator-toree/blob/master/Dockerfile
1717

18+
19+
# Todo: Add additional spark configuration:
20+
# https://spark.apache.org/docs/latest/configuration.html
21+
# TODO start spark master?
22+
# https://medium.com/@marcovillarreal_40011/creating-a-spark-standalone-cluster-with-docker-and-docker-compose-ba9d743a157f
23+
# ENV SPARK_MASTER_PORT 7077
24+
# ENV SPARK_MASTER_WEBUI_PORT 8080
25+
# ENV SPARK_WORKER_WEBUI_PORT 8081
26+
# ENV SPARK_MASTER_LOG /spark/logs
27+
# ENV SPARK_WORKER_LOG /spark/logs
28+
# export SPARK_MASTER_HOST=`hostname`
29+
# SPARK_WORKER_CORES=1
30+
# SPARK_WORKER_MEMORY=1G
31+
# SPARK_DRIVER_MEMORY=128m
32+
# SPARK_EXECUTOR_MEMORY=256m
33+
34+
# TODO configure spark ui to be proxied with base path:
35+
# https://stackoverflow.com/questions/45971127/wrong-css-location-of-spark-application-ui
36+
# https://github.com/jupyterhub/jupyter-server-proxy/issues/57
37+
# https://github.com/yuvipanda/jupyter-sparkui-proxy/blob/master/jupyter_sparkui_proxy/__init__.py
38+
39+
1840
# Install scala 2.12
1941
if [[ ! $(scala -version 2>&1) =~ "version 2.12" ]]; then
2042
# Update to Scala 2.12 is required for spark
21-
SCALA_VERSION=2.12.12
22-
echo "Updating to Scala $SCALA_VERSION. Please wait..."
23-
apt-get remove scala-library scala
24-
apt-get autoremove
25-
wget -q https://downloads.lightbend.com/scala/$SCALA_VERSION/scala-$SCALA_VERSION.deb -O ./scala.deb
26-
dpkg -i scala.deb
27-
rm scala.deb
28-
apt-get update
29-
apt-get install scala
43+
echo "Scala 2.12 is not installed. You should consider running the scala-utils.sh tool installer before continuing."
44+
sleep 10
3045
else
3146
echo "Scala 2.12 already installed."
3247
fi
@@ -38,7 +53,8 @@ if [ ! -d "$SPARK_HOME" ]; then
3853
cd $RESOURCES_PATH
3954
SPARK_VERSION="3.0.1"
4055
HADOOP_VERSION="3.2"
41-
wget https://mirror.checkdomain.de/apache/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop$HADOOP_VERSION.tgz -O ./spark.tar.gz
56+
echo "Downloading. Please wait..."
57+
wget -q https://mirror.checkdomain.de/apache/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop$HADOOP_VERSION.tgz -O ./spark.tar.gz
4258
tar xzf spark.tar.gz
4359
mv spark-$SPARK_VERSION-bin-hadoop$HADOOP_VERSION/ $SPARK_HOME
4460
rm spark.tar.gz
@@ -55,12 +71,12 @@ if [ ! -d "$SPARK_HOME" ]; then
5571
pip install --no-cache-dir sparkmagic
5672
jupyter serverextension enable --py sparkmagic
5773

58-
# Install sparkmonitor: https://github.com/krishnan-r/sparkmonitor
59-
pip install --no-cache-dir sparkmonitor
60-
jupyter nbextension install sparkmonitor --py --sys-prefix --symlink
61-
jupyter nbextension enable sparkmonitor --py --sys-prefix
62-
jupyter serverextension enable --py --sys-prefix sparkmonitor
63-
ipython profile create && echo "c.InteractiveShellApp.extensions.append('sparkmonitor.kernelextension')" >> $(ipython profile locate default)/ipython_kernel_config.py
74+
# TODO: does not work right now: Install sparkmonitor: https://github.com/krishnan-r/sparkmonitor
75+
# pip install --no-cache-dir sparkmonitor
76+
# jupyter nbextension install sparkmonitor --py --sys-prefix --symlink
77+
# jupyter nbextension enable sparkmonitor --py --sys-prefix
78+
# jupyter serverextension enable --py --sys-prefix sparkmonitor
79+
# ipython profile create && echo "c.InteractiveShellApp.extensions.append('sparkmonitor.kernelextension')" >> $(ipython profile locate default)/ipython_kernel_config.py
6480

6581
# Deprecated: jupyter-spark: https://github.com/mozilla/jupyter-spark
6682
# jupyter serverextension enable --py jupyter_spark && \
@@ -85,8 +101,10 @@ pip install --no-cache-dir pyspark findspark pyarrow spylon-kernel
85101
if [[ ! $(jupyter kernelspec list) =~ "toree" ]]; then
86102
echo "Installing Toree Kernel for Jupyter. Please wait..."
87103
TOREE_VERSION=0.5.0
88-
pip install --no-cache-dir https://dist.apache.org/repos/dist/dev/incubator/toree/$TOREE_VERSION-incubating-rc1/toree-pip/toree-$TOREE_VERSION.tar.gz
89-
jupyter toree install --sys-prefix --spark_home=$SPARK_HOME
104+
echo "Torre Kernel does not seem to work with the installed spark and scala verison."
105+
# TODO: Fix installation
106+
# pip install --no-cache-dir https://dist.apache.org/repos/dist/dev/incubator/toree/$TOREE_VERSION-incubating-rc1/toree-pip/toree-$TOREE_VERSION.tar.gz
107+
# jupyter toree install --sys-prefix --spark_home=$SPARK_HOME
90108
else
91109
echo "Toree Kernel for Jupyter is already installed."
92110
fi

0 commit comments

Comments
 (0)