Default profiles are on by default (apache-spark-on-k8s#486)

This avoids annoying issues with IDE integration and building where you have to remember incantation to run correct combination. This entails following changes

* Hadoop default changed from 2.7.4 to 2.9.1
* yarn, kubernetes, hadoop-cloud and kinesis modules are on by default

SparkR is left out since it requries a bit more invasive changes to enable by default

Author: robert3005

.circleci/config.yml

@@ -149,7 +149,7 @@ jobs:
           keys:
             - build-binaries-{{ checksum "build/mvn" }}-{{ checksum "build/sbt" }}
             - build-binaries-
-      - run: ./build/mvn -DskipTests -Phadoop-cloud -Phadoop-palantir -Pkinesis-asl -Pkubernetes -Pyarn -Psparkr install
+      - run: ./build/mvn -DskipTests -Psparkr install
       # Get sbt to run trivially, ensures its launcher is downloaded under build/
       - run: ./build/sbt -h || true
       - save_cache:

assembly/pom.xml

@@ -79,7 +79,29 @@
       <artifactId>spark-avro_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
     </dependency>
-
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-hadoop-cloud_${scala.binary.version}</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <!--
+    Redeclare this dependency to force it into the distribution.
+    -->
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-util</artifactId>
+      <scope>${hadoop.deps.scope}</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-yarn_${scala.binary.version}</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.spark</groupId>
+      <artifactId>spark-kubernetes_${scala.binary.version}</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <!--
       Because we don't shade dependencies anymore, we need to restore Guava to compile scope so
       that the libraries Spark depend on have it available. We'll package the version that Spark

dev/deps/spark-deps-hadoop-palantir

@@ -19,6 +19,7 @@ avro-1.8.2.jar
 avro-ipc-1.8.2.jar
 avro-mapred-1.8.2-hadoop2.jar
 aws-java-sdk-bundle-1.11.201.jar
+azure-data-lake-store-sdk-2.2.3.jar
 azure-keyvault-core-0.8.0.jar
 azure-storage-5.4.0.jar
 breeze-macros_2.12-0.13.2.jar
@@ -66,6 +67,7 @@ hadoop-annotations-2.9.1-palantir.11.jar
 hadoop-auth-2.9.1-palantir.11.jar
 hadoop-aws-2.9.1-palantir.11.jar
 hadoop-azure-2.9.1-palantir.11.jar
+hadoop-azure-datalake-2.9.1-palantir.11.jar
 hadoop-client-2.9.1-palantir.11.jar
 hadoop-common-2.9.1-palantir.11.jar
 hadoop-hdfs-client-2.9.1-palantir.11.jar

dev/mima

@@ -24,7 +24,7 @@ set -e
 FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
 cd "$FWDIR"
 
-SPARK_PROFILES="-Phadoop-palantir -Pmesos -Phadoop-cloud -Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive"
+SPARK_PROFILES="-Pmesos -Pkubernetes -Pyarn -Pspark-ganglia-lgpl -Pkinesis-asl -Phive-thriftserver -Phive"
 TOOLS_CLASSPATH="$(build/sbt -DcopyDependencies=false "export tools/fullClasspath" | tail -n1)"
 OLD_DEPS_CLASSPATH="$(build/sbt -DcopyDependencies=false $SPARK_PROFILES "export oldDeps/fullClasspath" | tail -n1)"
 

dev/publish-local.sh

@@ -1,27 +1,14 @@
 #!/usr/bin/env bash
 
 set -euo pipefail
-version=$(git describe --tags --first-parent)
-
-PALANTIR_FLAGS=(-Phadoop-cloud -Phadoop-palantir -Pkinesis-asl -Pkubernetes -Pyarn -Psparkr)
 
+FWDIR="$(cd "`dirname "${BASH_SOURCE[0]}"`"; pwd)"
 MVN_LOCAL=~/.m2/repository
 
-publish_artifacts() {
-  ./build/mvn versions:set -DnewVersion=$version
-  ./build/mvn -DskipTests "${PALANTIR_FLAGS[@]}" install
-}
+source "$FWDIR/publish_functions.sh"
 
-make_dist() {
-  build_flags="$1"
-  shift 1
-  hadoop_name="hadoop-palantir"
-  artifact_name="spark-dist_2.12-${hadoop_name}"
-  file_name="spark-dist-${version}-${hadoop_name}.tgz"
-  ./dev/make-distribution.sh --name "hadoop-palantir" --tgz "$@" $build_flags
-  mkdir -p $MVN_LOCAL/org/apache/spark/${artifact_name}/${version} && \
-  cp $file_name $MVN_LOCAL/org/apache/spark/${artifact_name}/${version}/${artifact_name}-${version}.tgz
-}
+set_version_and_install
+DONT_BUILD=true make_dist
+mkdir -p $MVN_LOCAL/org/apache/spark/${artifact_name}/${version}
+cp $file_name $MVN_LOCAL/org/apache/spark/${artifact_name}/${version}/${artifact_name}-${version}.tgz
 
-publish_artifacts
-make_dist "${PALANTIR_FLAGS[*]}"

dev/publish.sh

@@ -7,4 +7,4 @@ FWDIR="$(cd "`dirname "${BASH_SOURCE[0]}"`"; pwd)"
 source "$FWDIR/publish_functions.sh"
 
 publish_artifacts | tee -a "/tmp/publish_artifacts.log"
-DONT_BUILD=true make_dist "${PALANTIR_FLAGS[*]}" | tee -a "/tmp/make-dist.log"
+DONT_BUILD=true make_dist_and_deploy | tee -a "/tmp/make-dist.log"

dev/publish_functions.sh

@@ -2,7 +2,7 @@
 
 set -euo pipefail
 
-PALANTIR_FLAGS=(-Phadoop-cloud -Phadoop-palantir -Pkinesis-asl -Pkubernetes -Pyarn -Psparkr)
+PALANTIR_FLAGS=(-Psparkr)
 
 get_version() {
   git describe --tags --first-parent
@@ -14,6 +14,12 @@ set_version_and_package() {
   ./build/mvn -DskipTests "${PALANTIR_FLAGS[@]}" package
 }
 
+set_version_and_install() {
+  version=$(get_version)
+  ./build/mvn versions:set -DnewVersion="$version"
+  ./build/mvn -DskipTests "${PALANTIR_FLAGS[@]}" install
+}
+
 publish_artifacts() {
   tmp_settings="tmp-settings.xml"
   echo "<settings><servers><server>" > $tmp_settings
@@ -25,13 +31,15 @@ publish_artifacts() {
 }
 
 make_dist() {
-  build_flags="$1"
-  shift 1
   version=$(get_version)
   hadoop_name="hadoop-palantir"
   artifact_name="spark-dist_2.12-${hadoop_name}"
   file_name="spark-dist-${version}-${hadoop_name}.tgz"
-  ./dev/make-distribution.sh --name "hadoop-palantir" --tgz "$@" $build_flags
+  ./dev/make-distribution.sh --name "hadoop-palantir" --tgz "$@" "${PALANTIR_FLAGS[@]}"
+}
+
+make_dist_and_deploy() {
+  make_dist
   curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -T "$file_name" "https://api.bintray.com/content/palantir/releases/spark/${version}/org/apache/spark/${artifact_name}/${version}/${artifact_name}-${version}.tgz"
   curl -u $BINTRAY_USERNAME:$BINTRAY_PASSWORD -X POST "https://api.bintray.com/content/palantir/releases/spark/${version}/publish"
 }

dev/sbt-checkstyle

@@ -21,7 +21,6 @@
 # with failure (either resolution or compilation); the "q" makes SBT quit.
 ERRORS=$(echo -e "q\n" \
     | build/sbt \
-        -Phadoop-palantir \
         -Pkinesis-asl \
         -Pmesos \
         -Pkubernetes \

dev/test-dependencies.sh

@@ -29,7 +29,7 @@ export LC_ALL=C
 # TODO: This would be much nicer to do in SBT, once SBT supports Maven-style resolution.
 
 # NOTE: These should match those in the release publishing script
-HADOOP2_MODULE_PROFILES="-Phadoop-cloud -Pkubernetes -Pkinesis-asl -Pyarn"
+HADOOP2_MODULE_PROFILES=""
 MVN="build/mvn"
 HADOOP_PROFILES=(
     hadoop-palantir

hadoop-cloud/pom.xml

@@ -138,7 +138,6 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-azure</artifactId>
-      <version>${hadoop.version}</version>
       <scope>${hadoop.deps.scope}</scope>
       <exclusions>
         <exclusion>
@@ -159,25 +158,14 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-azure-datalake</artifactId>
+        <scope>${hadoop.deps.scope}</scope>
+    </dependency>
   </dependencies>
 
   <profiles>
-    <profile>
-        <id>hadoop-palantir</id>
-        <dependencies>
-            <dependency>
-                <groupId>org.apache.hadoop</groupId>
-                <artifactId>hadoop-azure-datalake</artifactId>
-                <scope>${hadoop.deps.scope}</scope>
-            </dependency>
-            <dependency>
-                <groupId>org.apache.hadoop</groupId>
-                <artifactId>hadoop-azure</artifactId>
-                <scope>${hadoop.deps.scope}</scope>
-            </dependency>
-        </dependencies>
-    </profile>
-
     <!--
      Hadoop 3 simplifies the classpath, and adds a new committer base class which
      enables store-specific committers.