expose MaxDirectMemorySize via env var

With newer versions of Netty, it appears `MaxDirectMemorySize` needs to be
increased for certain uses cases

This change exposes an env var, `MAX_DIRECT_MEMORY`, which allows for setting
this value

Author: Addison Higham

bin/manager_start.sh

@@ -22,7 +22,7 @@ GC_OPTS="-XX:+UseConcMarkSweepGC
          -XX:MaxPermSize=512m
          -XX:+CMSClassUnloadingEnabled "
 
-JAVA_OPTS="-XX:MaxDirectMemorySize=512M
+JAVA_OPTS="-XX:MaxDirectMemorySize=$MAX_DIRECT_MEMORY
            -XX:+HeapDumpOnOutOfMemoryError -Djava.net.preferIPv4Stack=true"
 
 MAIN="spark.jobserver.JobManager"

bin/server_start.sh

@@ -28,7 +28,7 @@ GC_OPTS="-XX:+UseConcMarkSweepGC
 # To truly enable JMX in AWS and other containerized environments, also need to set
 # -Djava.rmi.server.hostname equal to the hostname in that environment.  This is specific
 # depending on AWS vs GCE etc.
-JAVA_OPTS="-XX:MaxDirectMemorySize=512M \
+JAVA_OPTS="-XX:MaxDirectMemorySize=$MAX_DIRECT_MEMORY \
            -XX:+HeapDumpOnOutOfMemoryError -Djava.net.preferIPv4Stack=true \
            -Dcom.sun.management.jmxremote.port=9999 \
            -Dcom.sun.management.jmxremote.rmi.port=9999 \

bin/setenv.sh

@@ -61,6 +61,10 @@ if [ -z "$JOBSERVER_MEMORY" ]; then
   JOBSERVER_MEMORY=1G
 fi
 
+if [ -z "$MAX_DIRECT_MEMORY" ]; then
+  MAX_DIRECT_MEMORY=512M
+fi
+
 # This needs to be exported for standalone mode so drivers can connect to the Spark cluster
 export SPARK_HOME
 export YARN_CONF_DIR

job-server/config/local.sh.template

@@ -12,6 +12,7 @@ LOG_DIR=/var/log/job-server
 PIDFILE=spark-jobserver.pid
 JOBSERVER_MEMORY=1G
 SPARK_VERSION=1.6.0
+MAX_DIRECT_MEMORY=512M
 SPARK_HOME=/home/spark/spark-1.6.0
 SPARK_CONF_DIR=$SPARK_HOME/conf
 # Only needed for Mesos deploys