Fix regressions (#195)

Author: kevinwallimann

component-archetype/src/main/resources/archetype-resources/pom.xml

@@ -32,14 +32,15 @@
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
         <!-- Versions -->
         <hyperdrive.version>${project.version}</hyperdrive.version>
+        <spark.version>${spark.version}</spark.version>
 
         <!-- Build -->
         <maven.jar.plugin.version>3.1.2</maven.jar.plugin.version>
         <scala.version>${scala.version}</scala.version>
+        <scala.compat.version>${scala.compat.version}</scala.compat.version>
         <scala.maven.plugin.version>${scala.maven.plugin.version}</scala.maven.plugin.version>
 
         <!-- Tests -->
-        <scala.compat.version>${scala.compat.version}</scala.compat.version>
         <scalatest.version>${scalatest.version}</scalatest.version>
         <scalatest.maven.version>${scalatest.maven.version}</scalatest.maven.version>
     </properties>
@@ -53,6 +54,19 @@
             <scope>provided</scope>
         </dependency>
 
+        <!-- Spark -->
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-core_\${scala.compat.version}</artifactId>
+            <version>\${spark.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-sql_\${scala.compat.version}</artifactId>
+            <version>\${spark.version}</version>
+            <scope>provided</scope>
+        </dependency>
         <!-- Tests -->
         <dependency>
             <groupId>org.scalatest</groupId>

component-archetype/src/main/resources/archetype-resources/src/main/scala/reader/mycomponent/MyStreamReaderImpl.scala

@@ -19,8 +19,7 @@ package ${package}.reader.mycomponent
 
 import org.apache.commons.configuration2.Configuration
 import org.apache.logging.log4j.LogManager
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.streaming.DataStreamReader
+import org.apache.spark.sql.{DataFrame, SparkSession}
 import za.co.absa.hyperdrive.ingestor.api.reader.{StreamReader, StreamReaderFactory, StreamReaderFactoryProvider}
 import za.co.absa.hyperdrive.ingestor.api.{HasComponentAttributes, PropertyMetadata}
 
@@ -30,7 +29,7 @@ import za.co.absa.hyperdrive.ingestor.api.{HasComponentAttributes, PropertyMetad
  */
 
 private[reader] class MyStreamReaderImpl() extends StreamReader {
-  override def read(spark: SparkSession): DataStreamReader = ???
+  override def read(spark: SparkSession): DataFrame = ???
 }
 
 object MyStreamReaderImpl extends StreamReaderFactory with MyStreamReaderImplAttributes {