Properly remove hive from modules (apache-spark-on-k8s#338)

Author: dansanduleac

.circleci/config.yml

@@ -85,7 +85,7 @@ jobs:
             - build-binaries-{{ checksum "build/mvn" }}-{{ checksum "build/sbt" }}
             - build-binaries-
       - run: |
-          ./build/mvn -T1C -DskipTests -Phadoop-cloud -Phadoop-palantir -Pkinesis-asl -Pkubernetes -Pyarn -Phive -Psparkr install \
+          ./build/mvn -T1C -DskipTests -Phadoop-cloud -Phadoop-palantir -Pkinesis-asl -Pkubernetes -Pyarn -Psparkr install \
             | tee -a "/tmp/mvn-install.log"
       - store_artifacts:
           path: /tmp/mvn-install.log

dev/deps/spark-deps-hadoop-palantir

@@ -1,14 +1,9 @@
-JavaEWAH-0.3.2.jar
 RoaringBitmap-0.6.43.jar
-ST4-4.0.4.jar
 activation-1.1.1.jar
 aircompressor-0.8.jar
-antlr-2.7.7.jar
-antlr-runtime-3.4.jar
 antlr4-runtime-4.7.jar
 aopalliance-1.0.jar
 aopalliance-repackaged-2.5.0-b32.jar
-apache-log4j-extras-1.2.17.jar
 apacheds-i18n-2.0.0-M15.jar
 apacheds-kerberos-codec-2.0.0-M15.jar
 api-asn1-api-1.0.0-M20.jar
@@ -27,12 +22,8 @@ aws-java-sdk-s3-1.11.45.jar
 base64-2.3.8.jar
 bcpkix-jdk15on-1.58.jar
 bcprov-jdk15on-1.58.jar
-bonecp-0.8.0.RELEASE.jar
 breeze-macros_2.11-0.13.2.jar
 breeze_2.11-0.13.2.jar
-calcite-avatica-1.2.0-incubating.jar
-calcite-core-1.2.0-incubating.jar
-calcite-linq4j-1.2.0-incubating.jar
 chill-java-0.8.4.jar
 chill_2.11-0.8.4.jar
 classmate-1.1.0.jar
@@ -45,7 +36,6 @@ commons-compiler-3.0.8.jar
 commons-compress-1.8.1.jar
 commons-configuration-1.6.jar
 commons-crypto-1.0.0.jar
-commons-dbcp-1.4.jar
 commons-digester-1.8.jar
 commons-httpclient-3.1.jar
 commons-io-2.4.jar
@@ -62,17 +52,12 @@ core-1.1.2.jar
 curator-client-2.7.1.jar
 curator-framework-2.7.1.jar
 curator-recipes-2.7.1.jar
-datanucleus-api-jdo-3.2.6.jar
-datanucleus-core-3.2.10.jar
-datanucleus-rdbms-3.2.9.jar
-derby-10.12.1.1.jar
 dropwizard-jackson-0.9.2.jar
 dropwizard-lifecycle-0.9.2.jar
 dropwizard-metrics-0.9.2.jar
 dropwizard-metrics-influxdb-1.1.8.jar
 dropwizard-util-0.9.2.jar
 dropwizard-validation-0.9.2.jar
-eigenbase-properties-1.1.5.jar
 fastutil-7.0.13.jar
 flatbuffers-1.2.0-3f79e055.jar
 generex-1.0.1.jar
@@ -134,12 +119,10 @@ javax.inject-1.jar
 javax.inject-2.5.0-b32.jar
 javax.servlet-api-3.1.0.jar
 javax.ws.rs-api-2.0.1.jar
-javolution-5.5.1.jar
 jaxb-api-2.2.2.jar
 jboss-logging-3.2.1.Final.jar
 jcip-annotations-1.0.jar
 jcl-over-slf4j-1.7.25.jar
-jdo-api-3.0.1.jar
 jersey-client-2.25.1.jar
 jersey-common-2.25.1.jar
 jersey-container-servlet-2.25.1.jar
@@ -154,23 +137,19 @@ jetty-sslengine-6.1.26.jar
 jetty-util-6.1.26.jar
 jmespath-java-1.0.jar
 joda-time-2.9.9.jar
-jodd-core-3.5.2.jar
 json-smart-1.1.1.jar
 json4s-ast_2.11-3.5.3.jar
 json4s-core_2.11-3.5.3.jar
 json4s-jackson_2.11-3.5.3.jar
 json4s-scalap_2.11-3.5.3.jar
 jsp-api-2.1.jar
 jsr305-3.0.1.jar
-jta-1.1.jar
 jtransforms-2.4.0.jar
 jul-to-slf4j-1.7.25.jar
 kryo-shaded-3.0.3.jar
 kubernetes-client-3.0.0.jar
 kubernetes-model-2.0.0.jar
 leveldbjni-all-1.8.jar
-libfb303-0.9.3.jar
-libthrift-0.9.3.jar
 log4j-1.2.17.jar
 logging-interceptor-3.9.1.jar
 lz4-java-1.4.0.jar
@@ -201,7 +180,6 @@ parquet-common-1.9.1-palantir3.jar
 parquet-encoding-1.9.1-palantir3.jar
 parquet-format-2.3.1.jar
 parquet-hadoop-1.9.1-palantir3.jar
-parquet-hadoop-bundle-1.6.0.jar
 parquet-jackson-1.9.1-palantir3.jar
 protobuf-java-2.5.0.jar
 py4j-0.10.6.jar
@@ -216,14 +194,11 @@ shapeless_2.11-2.3.2.jar
 slf4j-api-1.7.25.jar
 slf4j-log4j12-1.7.25.jar
 snakeyaml-1.15.jar
-snappy-0.2.jar
 snappy-java-1.1.7.1.jar
 spire-macros_2.11-0.13.0.jar
 spire_2.11-0.13.0.jar
 stax-api-1.0-2.jar
-stax-api-1.0.1.jar
 stream-2.7.0.jar
-stringtemplate-3.2.1.jar
 univocity-parsers-2.5.9.jar
 validation-api-1.1.0.Final.jar
 xbean-asm5-shaded-4.4.jar

dev/publish-local.sh

@@ -3,7 +3,7 @@
 set -euo pipefail
 version=$(git describe --tags --first-parent)
 
-PALANTIR_FLAGS=(-Phadoop-cloud -Phadoop-palantir -Pkinesis-asl -Pkubernetes -Phive -Pyarn -Psparkr)
+PALANTIR_FLAGS=(-Phadoop-cloud -Phadoop-palantir -Pkinesis-asl -Pkubernetes -Pyarn -Psparkr)
 
 MVN_LOCAL="~/.m2/repository"
 

dev/publish_functions.sh

@@ -2,7 +2,7 @@
 
 set -euo pipefail
 
-PALANTIR_FLAGS=(-Phadoop-cloud -Phadoop-palantir -Pkinesis-asl -Pkubernetes -Phive -Pyarn -Psparkr)
+PALANTIR_FLAGS=(-Phadoop-cloud -Phadoop-palantir -Pkinesis-asl -Pkubernetes -Pyarn -Psparkr)
 
 get_version() {
   git describe --tags --first-parent

dev/sparktestsupport/modules.py

@@ -124,28 +124,28 @@ def __hash__(self):
 )
 
 
-hive = Module(
-    name="hive",
-    dependencies=[sql],
-    source_file_regexes=[
-        "sql/hive/",
-        "bin/spark-sql",
-    ],
-    build_profile_flags=[
-        "-Phive",
-    ],
-    sbt_test_goals=[
-        "hive/test",
-    ],
-    test_tags=[
-        "org.apache.spark.tags.ExtendedHiveTest"
-    ]
-)
+# hive = Module(
+#     name="hive",
+#     dependencies=[sql],
+#     source_file_regexes=[
+#         "sql/hive/",
+#         "bin/spark-sql",
+#     ],
+#     build_profile_flags=[
+#         "-Phive",
+#     ],
+#     sbt_test_goals=[
+#         "hive/test",
+#     ],
+#     test_tags=[
+#         "org.apache.spark.tags.ExtendedHiveTest"
+#     ]
+# )
 
 
 repl = Module(
     name="repl",
-    dependencies=[hive],
+    dependencies=[],
     source_file_regexes=[
         "repl/",
     ],
@@ -351,7 +351,7 @@ def __hash__(self):
 
 examples = Module(
     name="examples",
-    dependencies=[graphx, mllib, streaming, hive],
+    dependencies=[graphx, mllib, streaming],
     source_file_regexes=[
         "examples/",
     ],
@@ -384,7 +384,7 @@ def __hash__(self):
 
 pyspark_sql = Module(
     name="pyspark-sql",
-    dependencies=[pyspark_core, hive],
+    dependencies=[pyspark_core],
     source_file_regexes=[
         "python/pyspark/sql"
     ],
@@ -486,7 +486,7 @@ def __hash__(self):
 
 sparkr = Module(
     name="sparkr",
-    dependencies=[hive, mllib],
+    dependencies=[mllib],
     source_file_regexes=[
         "R/",
     ],

dev/test-dependencies.sh

@@ -29,7 +29,7 @@ export LC_ALL=C
 # TODO: This would be much nicer to do in SBT, once SBT supports Maven-style resolution.
 
 # NOTE: These should match those in the release publishing script
-HADOOP2_MODULE_PROFILES="-Phadoop-cloud -Pkubernetes -Pyarn -Pflume -Phive"
+HADOOP2_MODULE_PROFILES="-Phadoop-cloud -Pkubernetes -Pkinesis-asl -Pyarn"
 MVN="build/mvn"
 HADOOP_PROFILES=(
     hadoop-palantir

dists/hadoop-palantir/pom.xml

@@ -195,10 +195,6 @@
       <artifactId>guava</artifactId>
       <scope>${hadoop.deps.scope}</scope>
     </dependency>
-    <dependency>
-      <groupId>org.apache.spark</groupId>
-      <artifactId>spark-hive_${scala.binary.version}</artifactId>
-    </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-yarn_${scala.binary.version}</artifactId>

project/SparkBuild.scala

@@ -559,8 +559,6 @@ object SQL {
 object Hive {
 
   lazy val settings = Seq(
-    // Skip all hive tests because we don't use hive.
-    definedTests in Test := Nil,
     // Specially disable assertions since some Hive tests fail them
     javaOptions in Test := (javaOptions in Test).value.filterNot(_ == "-ea"),
     // Supporting all SerDes requires us to depend on deprecated APIs, so we turn off the warnings

python/pyspark/sql/readwriter.py

@@ -954,7 +954,7 @@ def _test():
     globs = pyspark.sql.readwriter.__dict__.copy()
     sc = SparkContext('local[4]', 'PythonTest')
     try:
-        spark = SparkSession.builder.enableHiveSupport().getOrCreate()
+        spark = SparkSession.builder.getOrCreate()
     except py4j.protocol.Py4JError:
         spark = SparkSession(sc)
 

python/pyspark/sql/tests.py

@@ -2923,6 +2923,7 @@ def test_create_dateframe_from_pandas_with_dst(self):
 
 class HiveSparkSubmitTests(SparkSubmitTests):
 
+    @unittest.skip("This distrbituion doesn't ship hive")
     def test_hivecontext(self):
         # This test checks that HiveContext is using Hive metastore (SPARK-16224).
         # It sets a metastore url and checks if there is a derby dir created by