Download Spark from mirrors; test vs newer Spark, libs (#90)

Author: srowen

.travis.yml

@@ -1,15 +1,29 @@
 language: python
-python:
-  - "2.7"
-  - "3.4"
-  - "3.5"
+
 cache:
   directories:
     - $HOME/.cache/spark-versions
-env:
-  matrix:
-    - SPARK_VERSION="2.1.1" SPARK_BUILD="spark-$SPARK_VERSION-bin-hadoop2.7" SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/$SPARK_BUILD.tgz"
-    - SPARK_VERSION="2.2.0" SPARK_BUILD="spark-$SPARK_VERSION-bin-hadoop2.7" SPARK_BUILD_URL="http://d3kbcqa49mib13.cloudfront.net/$SPARK_BUILD.tgz"
+
+matrix:
+  include:
+    - python: "2.7"
+      env:
+        - SPARK_VERSION=2.3.2
+        - NUMPY_VERSION=1.11.1
+        - PANDAS_VERSION=0.19.2
+        - SCIKIT_VERSION=0.18.1
+    - python: "3.5"
+      env:
+        - SPARK_VERSION=2.3.2
+        - NUMPY_VERSION=1.11.1
+        - PANDAS_VERSION=0.19.2
+        - SCIKIT_VERSION=0.18.1
+    - python: "3.6"
+      env:
+        - SPARK_VERSION=2.4.0
+        - NUMPY_VERSION=1.14.3
+        - PANDAS_VERSION=0.23.0
+        - SCIKIT_VERSION=0.19.1
 
 before_install:
  - ./bin/download_travis_dependencies.sh
@@ -20,9 +34,9 @@ install:
   # We do this conditionally because it saves us some downloading if the
   # version is the same.
   - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
-      wget https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh -O miniconda.sh;
+      curl -s https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh > miniconda.sh;
     else
-      wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
+      curl -s https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh > miniconda.sh;
     fi
   - bash miniconda.sh -b -p $HOME/miniconda
   - export PATH="$HOME/miniconda/bin:$PATH"
@@ -33,8 +47,8 @@ install:
   - conda info -a
 
   # Replace dep1 dep2 ... with your dependencies
-  - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION scikit-learn==0.18.1 nose=1.3.7 pandas=0.18
+  - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION nose==1.3.7 numpy==$NUMPY_VERSION pandas==$PANDAS_VERSION scikit-learn==$SCIKIT_VERSION
   - source activate test-environment
 
 script:
-  - SPARK_HOME=$HOME/.cache/spark-versions/$SPARK_BUILD ./python/run-tests.sh
+  - SPARK_HOME="$HOME/.cache/spark-versions/spark-$SPARK_VERSION-bin-hadoop2.7" ./python/run-tests.sh

bin/download_travis_dependencies.sh

@@ -1,17 +1,38 @@
+#!/usr/bin/env bash
+
 echo "Downloading Spark if necessary"
 echo "Spark version = $SPARK_VERSION"
-echo "Spark build = $SPARK_BUILD"
-echo "Spark build URL = $SPARK_BUILD_URL"
-mkdir -p $HOME/.cache/spark-versions
-tarfilename="$HOME/.cache/spark-versions/$SPARK_BUILD.tgz"
-dirname="$HOME/.cache/spark-versions/$SPARK_BUILD"
-if ! [ -d $dirname ]; then
-    echo "Missing $dirname, downloading archive"
-    echo `which curl`
-    curl "$SPARK_BUILD_URL" > $tarfilename
-    tar xvf $tarfilename --directory $HOME/.cache/spark-versions > /dev/null
-    echo "Content of directory:"
-    ls -la $HOME/.cache/spark-versions/
+
+sparkVersionsDir="$HOME/.cache/spark-versions"
+mkdir -p "$sparkVersionsDir"
+sparkBuild="spark-$SPARK_VERSION-bin-hadoop2.7"
+sparkBuildDir="$sparkVersionsDir/$sparkBuild"
+
+if [[ -d "$sparkBuildDir" ]]; then
+    echo "Skipping download - found Spark dir $sparkBuildDir"
 else
-    echo "Skipping download - found spark dir $dirname"
+    echo "Missing $sparkBuildDir, downloading archive"
+
+    # Get a local ASF mirror, as HTTPS
+    function apache_mirror() {
+        local mirror_url=$(curl -s https://www.apache.org/dyn/closer.lua?preferred=true)
+        echo "${mirror_url/http:/https:}"
+    }
+    # Try mirrors then fall back to dist.apache.org
+    for mirror in $(apache_mirror) $(apache_mirror) $(apache_mirror) https://dist.apache.org/repos/dist/release/; do
+        # If not already found,
+        if ! [[ -d "$sparkBuildDir" ]]; then
+            sparkURL="$mirror/spark/spark-$SPARK_VERSION/$sparkBuild.tgz"
+            echo "Downloading $sparkURL ..."
+            # Test whether it's reachable
+            if curl -s -I -f -o /dev/null "$sparkURL"; then
+                curl -s "$sparkURL" | tar xz --directory "$sparkVersionsDir"
+            else
+                echo "Could not reach $sparkURL"
+            fi
+        fi
+    done
+
+    echo "Content of $sparkBuildDir:"
+    ls -la "$sparkBuildDir"
 fi

project/build.properties

@@ -1,2 +1,2 @@
 // This file should only contain the version of sbt to use.
-sbt.version=0.13.6
+sbt.version=0.13.17

python/requirements.txt

@@ -1,2 +1,2 @@
 # This file should list any python package dependencies.
-scikit-learn>=0.18.1, <=0.19
+scikit-learn>=0.18.1

python/spark_sklearn/keyed_models.py

@@ -38,7 +38,8 @@
 <BLANKLINE>
 >>> km = KeyedEstimator(sklearnEstimator=LinearRegression(), yCol="y").fit(df)
 >>> def printFloat(x):
-...     return "{:.2f}".format(round(x, 2))
+...     rounded = round(x, 2)
+...     return "{:.2f}".format(0 if rounded == 0 else rounded)
 ...
 >>> def printModel(model):
 ...     coef = "[" + ", ".join(map(printFloat, model.coef_)) + "]"