Removes without-hadoop dist and publishes dist tgz to the same location as the pom (apache-spark-on-k8s#233)

Author: robert3005

dev/lint-java

@@ -20,7 +20,7 @@
 SCRIPT_DIR="$( cd "$( dirname "$0" )" && pwd )"
 SPARK_ROOT_DIR="$(dirname $SCRIPT_DIR)"
 
-ERRORS=$($SCRIPT_DIR/../build/mvn -Pkubernetes -Pkinesis-asl -Pmesos -Pyarn -Phive -Phive-thriftserver checkstyle:check | grep ERROR)
+ERRORS=$($SCRIPT_DIR/../build/mvn -Phadoop-cloud -Pkubernetes -Pkinesis-asl -Pmesos -Pyarn -Phive -Phive-thriftserver checkstyle:check | grep ERROR)
 
 if test ! -z "$ERRORS"; then
     echo -e "Checkstyle checks failed at following occurrences:\n$ERRORS"

dev/make-distribution.sh

@@ -142,7 +142,7 @@ fi
 echo "Spark version is $VERSION"
 VERSION_SET=$("$MVN" versions:set -DnewVersion=$VERSION | tail -n 1)
 if [ "$MAKE_TGZ" == "true" ]; then
-  echo "Making spark-dist-$VERSION-$NAME.tgz"
+  echo "Making spark-dist_2.11-$NAME-$VERSION.tgz"
 else
   echo "Making distribution for Spark $VERSION in '$DISTDIR'..."
 fi

dev/publish-local.sh

@@ -1,31 +1,26 @@
 #!/usr/bin/env bash
 
 set -euo pipefail
-VERSION=$(git describe --tags)
-HADOOP_VERSION=$(./build/mvn help:evaluate -Phadoop-palantir -Dexpression=hadoop.version 2>/dev/null\
-  | grep -v "INFO"\
-  | tail -n 1)
-
+version=$(git describe --tags)
 
 PALANTIR_FLAGS=(-Phadoop-cloud -Phadoop-palantir -Pkinesis-asl -Pkubernetes -Phive -Pyarn -Psparkr)
 
 MVN_LOCAL="~/.m2/repository"
 
 publish_artifacts() {
-  ./build/mvn versions:set -DnewVersion=$VERSION
+  ./build/mvn versions:set -DnewVersion=$version
   ./build/mvn -DskipTests "${PALANTIR_FLAGS[@]}" install clean
 }
 
 make_dist() {
-  dist_name="$1"
-  build_flags="$2"
-  shift 2
-  dist_version="${VERSION}-${dist_name}"
-  file_name="spark-dist-${dist_version}.tgz"
-  ./dev/make-distribution.sh --name $dist_name --tgz "$@" $build_flags
-  mkdir -p $MVN_LOCAL/org/apache/spark/spark-dist/${dist_version} && \
-  cp $file_name $MVN_LOCAL/org/apache/spark/spark-dist/${dist_version}/
+  build_flags="$1"
+  shift 1
+  artifact_name="spark-dist_2.11-hadoop-palantir"
+  file_name="${artifact_name}-${version}.tgz"
+  ./dev/make-distribution.sh --name "hadoop-palantir" --tgz "$@" $build_flags
+  mkdir -p $MVN_LOCAL/org/apache/spark/${artifact_name}/${version} && \
+  cp $file_name $MVN_LOCAL/org/apache/spark/${artifact_name}/${version}/
 }
 
 publish_artifacts
-make_dist hadoop-$HADOOP_VERSION "${PALANTIR_FLAGS[*]}"
+make_dist "${PALANTIR_FLAGS[*]}"

dev/publish.sh

@@ -17,15 +17,13 @@ publish_artifacts() {
 }
 
 make_dist() {
-  dist_name="$1"
-  build_flags="$2"
-  shift 2
-  dist_version="${version}-${dist_name}"
-  file_name="spark-dist-${dist_version}.tgz"
-  ./dev/make-distribution.sh --name $dist_name --tgz "$@" $build_flags
-  curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -T $file_name "https://api.bintray.com/content/palantir/releases/spark/${version}/org/apache/spark/spark-dist/${dist_version}/${file_name}"
+  build_flags="$1"
+  shift 1
+  artifact_name="spark-dist_2.11-hadoop-palantir"
+  file_name="${artifact_name}-${version}.tgz"
+  ./dev/make-distribution.sh --name "hadoop-palantir" --tgz "$@" $build_flags
+  curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -T $file_name "https://api.bintray.com/content/palantir/releases/spark/${version}/org/apache/spark/${artifact_name}/${version}/${file_name}"
 }
 
 publish_artifacts
-make_dist hadoop-2.8.0-palantir.8 "${PALANTIR_FLAGS[*]}" --clean
-make_dist without-hadoop "-Phadoop-provided -Pkubernetes -Phive -Pyarn -Psparkr" --clean
+make_dist "${PALANTIR_FLAGS[*]}" --clean

dev/sparktestsupport/modules.py

@@ -521,6 +521,14 @@ def __hash__(self):
     sbt_test_goals=["kubernetes/test"]
 )
 
+cloud = Module(
+    name="hadoop-cloud",
+    dependencies=[],
+    source_file_regexes=["hadoop-cloud/"],
+    build_profile_flags=["-Phadoop-cloud"],
+    sbt_test_goals=["hadoop-cloud/test"]
+)
+
 # The root module is a dummy module which is used to run all of the tests.
 # No other modules should directly depend on this module.
 root = Module(

dists/without-hadoop/pom.xml

@@ -107,7 +107,6 @@
     <module>external/kafka-0-10</module>
     <module>external/kafka-0-10-assembly</module>
     <module>external/kafka-0-10-sql</module>
-    <module>dists/without-hadoop</module>
     <module>dists/hadoop-palantir</module>
   </modules>