Ignore flaky scala tests as well as hive tests (apache-spark-on-k8s#335)

Author: dansanduleac

common/tags/src/test/java/org/apache/spark/tags/FlakyTest.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.tags;
+
+import org.scalatest.TagAnnotation;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@TagAnnotation
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.METHOD, ElementType.TYPE})
+public @interface FlakyTest { }

common/tags/src/test/scala/org/apache/spark/tags/Flaky.scala

@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.tags
+
+import org.scalatest.Tag
+
+object Flaky extends Tag(classOf[FlakyTest].getCanonicalName)

dev/run-scala-tests.py

@@ -37,5 +37,9 @@ def java_version():
     env = get_build_environment()
     mtt = modules_to_test(env)
 
+    excluded_tags = mtt.excluded_tags + [
+        "org.apache.spark.tags.FlakyTest"
+    ]
+
     # run the test suites
-    run_scala_tests(env.build_tool, env.hadoop_version, mtt.test_modules, mtt.excluded_tags)
+    run_scala_tests(env.build_tool, env.hadoop_version, mtt.test_modules, excluded_tags)

project/CirclePlugin.scala

@@ -46,9 +46,9 @@ object CirclePlugin extends AutoPlugin {
   override def trigger: PluginTrigger = allRequirements
 
   private[this] lazy val testsByProject = Def.task {
-    // Defaults.detectTests is basically the value of Keys.definedTests, but since we're
-    // overriding the latter depending on the value of this task, we can't depend on it
-    ProjectTests(thisProjectRef.value, Defaults.detectTests.value)
+    // We can use Keys.definedTests because we resolve this task 'in Test' but later define it
+    // 'in Circle' so there's no cycle.
+    ProjectTests(thisProjectRef.value, (definedTests in Test).value)
   }
 
   private[this] lazy val mapper = new ObjectMapper() with ScalaObjectMapper

project/SparkBuild.scala

@@ -559,6 +559,8 @@ object SQL {
 object Hive {
 
   lazy val settings = Seq(
+    // Skip all hive tests because we don't use hive.
+    definedTests in Test := Nil,
     // Specially disable assertions since some Hive tests fail them
     javaOptions in Test := (javaOptions in Test).value.filterNot(_ == "-ea"),
     // Supporting all SerDes requires us to depend on deprecated APIs, so we turn off the warnings

sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala

@@ -25,6 +25,7 @@ import org.scalatest.BeforeAndAfterAll
 import org.apache.spark.SparkException
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSQLContext
+import org.apache.spark.tags.Flaky
 
 
 class FileBasedDataSourceSuite extends QueryTest with SharedSQLContext with BeforeAndAfterAll {
@@ -115,7 +116,11 @@ class FileBasedDataSourceSuite extends QueryTest with SharedSQLContext with Befo
   }
 
   allFileBasedDataSources.foreach { format =>
-    testQuietly(s"Enabling/disabling ignoreMissingFiles using $format") {
+    val tags = format match {
+      case "orc" => List(Flaky)
+      case _ => List()
+    }
+    testQuietly(s"Enabling/disabling ignoreMissingFiles using $format", tags: _*) {
       def testIgnoreMissingFiles(): Unit = {
         withTempDir { dir =>
           val basePath = dir.getCanonicalPath

sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamingJoinSuite.scala

@@ -35,6 +35,7 @@ import org.apache.spark.sql.execution.streaming.{MemoryStream, StatefulOperatorS
 import org.apache.spark.sql.execution.streaming.state.{StateStore, StateStoreProviderId}
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.types._
+import org.apache.spark.tags.Flaky
 import org.apache.spark.util.Utils
 
 
@@ -446,7 +447,7 @@ class StreamingOuterJoinSuite extends StreamTest with StateStoreMetricsTest with
     (input1, input2, joined)
   }
 
-  test("left outer early state exclusion on left") {
+  test("left outer early state exclusion on left", Flaky) {
     val (leftInput, df1) = setupStream("left", 2)
     val (rightInput, df2) = setupStream("right", 3)
     // Use different schemas to ensure the null row is being generated from the correct side.
@@ -477,7 +478,7 @@ class StreamingOuterJoinSuite extends StreamTest with StateStoreMetricsTest with
     )
   }
 
-  test("left outer early state exclusion on right") {
+  test("left outer early state exclusion on right", Flaky) {
     val (leftInput, df1) = setupStream("left", 2)
     val (rightInput, df2) = setupStream("right", 3)
     // Use different schemas to ensure the null row is being generated from the correct side.
@@ -508,7 +509,7 @@ class StreamingOuterJoinSuite extends StreamTest with StateStoreMetricsTest with
     )
   }
 
-  test("right outer early state exclusion on left") {
+  test("right outer early state exclusion on left", Flaky) {
     val (leftInput, df1) = setupStream("left", 2)
     val (rightInput, df2) = setupStream("right", 3)
     // Use different schemas to ensure the null row is being generated from the correct side.
@@ -539,7 +540,7 @@ class StreamingOuterJoinSuite extends StreamTest with StateStoreMetricsTest with
     )
   }
 
-  test("right outer early state exclusion on right") {
+  test("right outer early state exclusion on right", Flaky) {
     val (leftInput, df1) = setupStream("left", 2)
     val (rightInput, df2) = setupStream("right", 3)
     // Use different schemas to ensure the null row is being generated from the correct side.

sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala

@@ -28,6 +28,7 @@ import scala.util.control.NonFatal
 
 import org.apache.hadoop.fs.Path
 import org.scalatest.{BeforeAndAfterAll, Suite}
+import org.scalatest.Tag
 import org.scalatest.concurrent.Eventually
 
 import org.apache.spark.SparkFunSuite
@@ -80,8 +81,8 @@ private[sql] trait SQLTestUtils extends SparkFunSuite with SQLTestUtilsBase with
    * System.out or System.err. Otherwise, ConsoleAppender will still output to the console even if
    * we change System.out and System.err.
    */
-  protected def testQuietly(name: String)(f: => Unit): Unit = {
-    test(name) {
+  protected def testQuietly(name: String, tags: Tag*)(f: => Unit): Unit = {
+    test(name, tags: _*) {
       quietly {
         f
       }