bump spark version (#150)

* bump spark version

* update default value of SPARK_HOME

Author: yogeshg

.travis.yml

@@ -16,9 +16,9 @@ cache:
 env:
   global:
     - SCALA_VERSION=2.11.8
-    - SPARK_VERSION=2.3.0
+    - SPARK_VERSION=2.3.1
     - SPARK_BUILD="spark-${SPARK_VERSION}-bin-hadoop2.7"
-    - SPARK_BUILD_URL="https://dist.apache.org/repos/dist/release/spark/spark-2.3.0/spark-2.3.0-bin-hadoop2.7.tgz"
+    - SPARK_BUILD_URL="https://dist.apache.org/repos/dist/release/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop2.7.tgz"
     - SPARK_HOME=$HOME/.cache/spark-versions/$SPARK_BUILD
     - RUN_ONLY_LIGHT_TESTS=True
   matrix:

bin/download_travis_dependencies.sh

@@ -7,9 +7,8 @@ mkdir -p $HOME/.cache/spark-versions
 filename="$HOME/.cache/spark-versions/$SPARK_BUILD.tgz"
 if ! [ -f $filename ]; then
 	echo "Downloading file..."
-	echo `which curl`
-	curl "$SPARK_BUILD_URL" > $filename
-	echo "Content of directory:"
-	ls -la $HOME/.cache/spark-versions/*
-	tar xvf $filename --directory $HOME/.cache/spark-versions > /dev/null
+	wget "$SPARK_BUILD_URL" -O $filename
+	echo "[Debug] Following should list a valid spark binary"
+	ls -larth $HOME/.cache/spark-versions/*
+	tar -xvzf $filename --directory $HOME/.cache/spark-versions > /dev/null
 fi

dev/run.py

@@ -125,8 +125,8 @@ def print_if(cond, *args):
 
 def _get_required_env(default=False, interactive=False, override=False, verbose=False):
     default_env = {'PYSPARK_PYTHON': 'python',
-                   'SPARK_VERSION': '2.3.0',
-                   'SPARK_HOME': os.path.join(HOME, 'bin/spark-2.3.0-bin-hadoop2.7/'),
+                   'SPARK_VERSION': '2.3.1',
+                   'SPARK_HOME': os.path.join(HOME, 'bin/spark-2.3.1-bin-hadoop2.7/'),
                    'SCALA_VERSION': '2.11.8'}
 
     if override and not (interactive or default):