Skip to content

Commit b731636

Browse files
RussellSpitzerRussellSpitzer
committed
Merge pull request #2 from riptano/DSP-5892-2
Jobserver Integration with DSE
2 parents e160404 + a8405dd commit b731636

File tree

7 files changed

+185
-15
lines changed

7 files changed

+185
-15
lines changed

‎bin/kill-process-tree.sh‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#!/bin/bash
2+
3+
if [ "$(which pgrep)" == "" ]; then
4+
echo "pgrep is not available" >&2
5+
exit 1
6+
fi
7+
8+
if [ "$#" != "2" ]; then
9+
echo "Syntax error, the expected usage is: kill-process-tree <signal> <process_id>" >&2
10+
exit 1
11+
fi
12+
13+
# This simple script allows to kill the whole process tree. The first argument is a signal and the second
14+
# is the root process
15+
parent=$2
16+
signal=$1
17+
18+
# A string which will be populated with descendant processes ids
19+
pids="$parent"
20+
21+
# Kills the given process if it is running. It also checks whether the given parameter contains process ID
22+
function kill_if_running() {
23+
if [[ "$1" =~ [0-9]+ ]]; then
24+
ps -e -o pid= | grep $1 > /dev/null
25+
if [ $? -eq 0 ]; then
26+
kill "-$signal" "$1"
27+
return 0
28+
else
29+
return 1
30+
fi
31+
fi
32+
}
33+
34+
# Populates pids with descendant processes ids
35+
function get_descendants_pids() {
36+
for cpid in $(pgrep -P $1);
37+
do
38+
pids="$cpid $pids"
39+
get_descendants_pids $cpid
40+
done
41+
}
42+
43+
get_descendants_pids $parent
44+
45+
for pid in $pids
46+
do
47+
if [ "$signal" == "" ]; then
48+
echo "$pid"
49+
else
50+
kill_if_running $pid
51+
if [ $? -eq 0 ]; then
52+
# If the process has been really killed, then wait for 3 seconds before going to the next
53+
# process. In most cases, where there is a scripts execution hierarchy with Java process
54+
# at the bottom, it is sufficient to kill that Java process and the scripts will just exit
55+
# in a natural way
56+
echo "Sent $signal to $pid, sleeping for 3 seconds"
57+
sleep 3s
58+
fi
59+
fi
60+
done

‎bin/server_package.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,9 @@ fi
4444
FILES="job-server-extras/target/scala-$majorVersion/spark-job-server.jar
4545
bin/server_start.sh
4646
bin/server_stop.sh
47+
bin/kill-process-tree.sh
4748
$CONFIG_DIR/$ENV.conf
48-
config/log4j-server.properties"
49+
config/logback-server.xml"
4950

5051
rm -rf $WORK_DIR
5152
mkdir -p $WORK_DIR

‎bin/server_start.sh‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ set -e
66

77
get_abs_script_path() {
88
pushd . >/dev/null
9-
cd $(dirname $0)
9+
cd "$(dirname "$0")"
1010
appdir=$(pwd)
1111
popd >/dev/null
1212
}
@@ -26,14 +26,14 @@ JAVA_OPTS="-XX:MaxDirectMemorySize=512M
2626

2727
MAIN="spark.jobserver.JobServer"
2828

29-
conffile=$(ls -1 $appdir/*.conf | head -1)
29+
conffile="$(ls -1 "$appdir"/*.conf | head -1)"
3030
if [ -z "$conffile" ]; then
3131
echo "No configuration file found"
3232
exit 1
3333
fi
3434

3535
if [ -f "$appdir/settings.sh" ]; then
36-
. $appdir/settings.sh
36+
. "$appdir/settings.sh"
3737
else
3838
echo "Missing $appdir/settings.sh, exiting"
3939
exit 1
@@ -46,7 +46,7 @@ fi
4646

4747
pidFilePath=$appdir/$PIDFILE
4848

49-
if [ -f "$pidFilePath" ] && kill -0 $(cat "$pidFilePath"); then
49+
if [ -f "$pidFilePath" ] && kill -0 "$(cat "$pidFilePath")"; then
5050
echo 'Job server is already running'
5151
exit 1
5252
fi
@@ -57,8 +57,9 @@ if [ -z "$LOG_DIR" ]; then
5757
fi
5858
mkdir -p $LOG_DIR
5959

60-
LOGGING_OPTS="-Dlog4j.configuration=file:$appdir/log4j-server.properties
61-
-DLOG_DIR=$LOG_DIR"
60+
LOGGING_OPTS="-DLOG_DIR=$LOG_DIR"
61+
62+
export SPARK_SUBMIT_LOGBACK_CONF_FILE="$appdir/logback-server.xml"
6263

6364
# For Mesos
6465
CONFIG_OVERRIDES=""
@@ -77,8 +78,9 @@ fi
7778
# This needs to be exported for standalone mode so drivers can connect to the Spark cluster
7879
export SPARK_HOME
7980

80-
$SPARK_HOME/bin/spark-submit --class $MAIN --driver-memory $DRIVER_MEMORY \
81+
# DSE_BIN is set in settings.sh
82+
"$DSE_HOME/bin/dse" spark-submit --class "$MAIN" --driver-memory 5G \
8183
--conf "spark.executor.extraJavaOptions=$LOGGING_OPTS" \
8284
--driver-java-options "$GC_OPTS $JAVA_OPTS $LOGGING_OPTS $CONFIG_OVERRIDES" \
83-
$@ $appdir/spark-job-server.jar $conffile 2>&1 &
84-
echo $! > $pidFilePath
85+
"$@" "$appdir/spark-job-server.jar" "$conffile" 2>&1 &
86+
echo "$!" > "$pidFilePath"

‎bin/server_stop.sh‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,26 +3,30 @@
33

44
get_abs_script_path() {
55
pushd . >/dev/null
6-
cd $(dirname $0)
6+
cd "$(dirname "$0")"
77
appdir=$(pwd)
88
popd >/dev/null
99
}
1010

1111
get_abs_script_path
1212

1313
if [ -f "$appdir/settings.sh" ]; then
14-
. $appdir/settings.sh
14+
. "$appdir/settings.sh"
1515
else
1616
echo "Missing $appdir/settings.sh, exiting"
1717
exit 1
1818
fi
1919

2020
pidFilePath=$appdir/$PIDFILE
2121

22-
if [ ! -f "$pidFilePath" ] || ! kill -0 $(cat "$pidFilePath"); then
22+
if [ ! -f "$pidFilePath" ] || ! kill -0 "$(cat "$pidFilePath")"; then
2323
echo 'Job server not running'
2424
else
2525
echo 'Stopping job server...'
26-
kill -15 $(cat "$pidFilePath") && rm -f "$pidFilePath"
26+
PID="$(cat "$pidFilePath")"
27+
"$(dirname "$0")"/kill-process-tree.sh 15 $PID && rm "$pidFilePath"
2728
echo '...job server stopped'
2829
fi
30+
31+
32+

‎job-server/config/dse.conf‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
# Template for a Spark Job Server configuration file
2+
# When deployed these settings are loaded when job server starts
3+
#
4+
# Spark Cluster / Job Server configuration
5+
spark {
6+
# Spark Master will be automatically learned via the DSE
7+
# spark.master will be passed to each job's JobContext
8+
# master = "local[4]"
9+
# master = "mesos://vm28-hulk-pub:5050"
10+
# master = "yarn-client"
11+
12+
# Default # of CPUs for jobs to use for Spark standalone cluster
13+
job-number-cpus = 4
14+
15+
jobserver {
16+
port = 8090
17+
jar-store-rootdir = /tmp/jobserver/jars
18+
19+
jobdao = spark.jobserver.io.JobFileDAO
20+
21+
filedao {
22+
rootdir = /tmp/spark-job-server/filedao/data
23+
}
24+
}
25+
26+
# predefined Spark contexts
27+
# contexts {
28+
# my-low-latency-context {
29+
# num-cpu-cores = 1 # Number of cores to allocate. Required.
30+
# memory-per-node = 512m # Executor memory per node, -Xmx style eg 512m, 1G, etc.
31+
# }
32+
# # define additional contexts here
33+
# }
34+
35+
# universal context configuration. These settings can be overridden, see README.md
36+
context-settings {
37+
num-cpu-cores = 2 # Number of cores to allocate. Required.
38+
memory-per-node = 512m # Executor memory per node, -Xmx style eg 512m, #1G, etc.
39+
40+
# in case spark distribution should be accessed from HDFS (as opposed to being installed on every mesos slave)
41+
# spark.executor.uri = "hdfs://namenode:8020/apps/spark/spark.tgz"
42+
43+
# uris of jars to be loaded into the classpath for this context. Uris is a string list, or a string separated by commas ','
44+
# dependent-jar-uris = ["file:///some/path/present/in/each/mesos/slave/somepackage.jar"]
45+
46+
# If you wish to pass any settings directly to the sparkConf as-is, add them here in passthrough,
47+
# such as hadoop connection settings that don't use the "spark." prefix
48+
passthrough {
49+
#es.nodes = "192.1.1.1"
50+
}
51+
}
52+
53+
# This needs to match SPARK_HOME for cluster SparkContexts to be created successfully
54+
# home = "/home/spark/spark"
55+
}
56+
57+
# Note that you can use this file to define settings not only for job server,
58+
# but for your Spark jobs as well. Spark job configuration merges with this configuration file as defaults.

‎job-server/config/dse.sh‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# DataStax Distribution Config
2+
# Environment and deploy file
3+
# For use with bin/server_deploy, bin/server_package etc.
4+
5+
APP_USER=cassandra
6+
APP_GROUP=cassandra
7+
8+
#Check Home Directory
9+
#Relative Tar Location and
10+
#Package location for dse-env.sh to get environment variables
11+
if [ -z "$DSE_ENV" ]; then
12+
for include in "$HOME/.dse-env.sh" \
13+
"`dirname "$0"`/../../../bin/dse-env.sh" \
14+
"/etc/dse/dse-env.sh"; do
15+
if [ -r "$include" ]; then
16+
DSE_ENV="$include"
17+
break
18+
fi
19+
done
20+
fi
21+
22+
#ENV is set for the build script server_package, If it isn't set then we need
23+
# to be able to read DSE_ENV to set Spark Env variables
24+
if [ -z "$DSE_ENV" ] && [ -z "$ENV" ]; then
25+
echo "DSE_ENV could not be determined."
26+
exit 1
27+
elif [ -r "$DSE_ENV" ]; then
28+
. "$DSE_ENV"
29+
elif [ -z "$ENV" ]; then
30+
echo "Location pointed by DSE_ENV not readable: $DSE_ENV"
31+
exit 1
32+
fi
33+
34+
SPARK_VERSION=1.3.1.0 #Last digit is DSE Specific
35+
36+
DEPLOY_HOSTS="localhost"
37+
38+
INSTALL_DIR="$DSE_COMPONENTS_ROOT/spark/spark-jobserver"
39+
LOG_DIR=/var/log/spark/job-server
40+
41+
PIDFILE=spark-jobserver.pid
42+
43+
SPARK_CONF_DIR=${SPARK_CONF_DIR:-"$SPARK_HOME/conf"}
44+
45+
SCALA_VERSION=2.10.5 # or 2.11.6

‎version.sbt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
version in ThisBuild := "0.5.2-SNAPSHOT"
1+
version in ThisBuild := "0.5.1.155"

0 commit comments

Comments
 (0)