[SPARK-25598][STREAMING][BUILD][TEST-MAVEN] Remove flume connector in Spark 3

Removes all vestiges of Flume in the build, for Spark 3.
I don't think this needs Jenkins config changes.

Existing tests.

Closes apache#22692 from srowen/SPARK-25598.

Authored-by: Sean Owen <[email protected]>
Signed-off-by: Sean Owen <[email protected]>

Author: srowen

build.gradle

@@ -115,7 +115,6 @@ allprojects {
     kafkaVersion = '2.2.2'
     scalaLoggingVersion = '3.9.0' // keep consistent with kafkaVersion
     joptVersion = '5.0.4'
-    flumeVersion = '1.8.0'
     db2JdbcVersion = '10.5.0.5'
     dockerClientVersion = '8.14.5'
     mysqlVersion = '8.0.13'
@@ -441,7 +440,6 @@ gradle.taskGraph.whenReady { graph ->
 task generateSources {
   dependsOn subprojectBase + 'snappy-spark-core_' + scalaBinaryVersion + ':generateBuildInfo'
   dependsOn subprojectBase + 'snappy-spark-catalyst_' + scalaBinaryVersion + ':generateGrammarSource'
-  dependsOn subprojectBase + 'snappy-spark-streaming-flume-sink_' + scalaBinaryVersion + ':generateAvroJava'
 }
 
 if (rootProject.name == 'snappy-spark') {

dev/run-tests.py

@@ -325,7 +325,6 @@ def build_spark_sbt(hadoop_version):
     build_profiles = get_hadoop_profiles(hadoop_version) + modules.root.build_profile_flags
     sbt_goals = ["test:package",  # Build test jars as some tests depend on them
                  "streaming-kafka-0-8-assembly/assembly",
-                 "streaming-flume-assembly/assembly",
                  "streaming-kinesis-asl-assembly/assembly"]
     profiles_and_goals = build_profiles + sbt_goals
 

dev/sparktestsupport/modules.py

@@ -267,39 +267,6 @@ def __hash__(self):
     ]
 )
 
-streaming_flume_sink = Module(
-    name="streaming-flume-sink",
-    dependencies=[streaming],
-    source_file_regexes=[
-        "external/flume-sink",
-    ],
-    sbt_test_goals=[
-        "streaming-flume-sink/test",
-    ]
-)
-
-
-streaming_flume = Module(
-    name="streaming-flume",
-    dependencies=[streaming],
-    source_file_regexes=[
-        "external/flume",
-    ],
-    sbt_test_goals=[
-        "streaming-flume/test",
-    ]
-)
-
-
-streaming_flume_assembly = Module(
-    name="streaming-flume-assembly",
-    dependencies=[streaming_flume, streaming_flume_sink],
-    source_file_regexes=[
-        "external/flume-assembly",
-    ]
-)
-
-
 mllib_local = Module(
     name="mllib-local",
     dependencies=[tags],
@@ -387,7 +354,6 @@ def __hash__(self):
         pyspark_core,
         streaming,
         streaming_kafka,
-        streaming_flume_assembly,
         streaming_kinesis_asl
     ],
     source_file_regexes=[

docs/streaming-custom-receivers.md

@@ -4,7 +4,7 @@ title: Spark Streaming Custom Receivers
 ---
 
 Spark Streaming can receive streaming data from any arbitrary data source beyond
-the ones for which it has built-in support (that is, beyond Flume, Kafka, Kinesis, files, sockets, etc.).
+the ones for which it has built-in support (that is, beyond Kafka, Kinesis, files, sockets, etc.).
 This requires the developer to implement a *receiver* that is customized for receiving data from
 the concerned data source. This guide walks through the process of implementing a custom receiver
 and using it in a Spark Streaming application. Note that custom receivers can be implemented