Feature/scala2.11 12 cross build (#193)

* Add scala 2.11 / scala 2.12 cross build

* Fix foreachBatch ambiguity with Scala 2.12

* Force version of paranamer

* Rename artefacts

* Remove directory maven plugin as it assumes a project dir based on the artefact id

* Fix archetype readme

* Try matrix build script

* Fix build name

* Add cross compile info to readme

* Fix readme

Author: kevinwallimann

.github/workflows/build.yml

@@ -1,4 +1,4 @@
-name: build
+name: Build
 
 on:
   pull_request:
@@ -7,12 +7,24 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
-
+    strategy:
+      fail-fast: false
+      matrix:
+        scala: [ 2.11, 2.12 ]
+    name: Scala ${{ matrix.scala }}, Spark 2.4
     steps:
       - uses: actions/checkout@v2
       - name: Set up JDK 1.8
         uses: actions/setup-java@v1
         with:
           java-version: 1.8
-      - name: Run tests
-        run: mvn clean test -Pall-tests
+      - uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-${{ matrix.scala }}-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.scala }}-
+      - name: Switch scala version
+        run: mvn scala-cross-build:change-version -Pscala-${{ matrix.scala }}
+      - name: Build and run tests
+        run: mvn clean test -Pscala-${{ matrix.scala }},all-tests

README.md

@@ -356,3 +356,22 @@ writer.kafka.schema.registry.url=${transformer.[avro.decoder].schema.registry.ur
 Hyperdrive ingestions may be triggered using the Workflow Manager, which is developed in a separate repository: https://github.com/AbsaOSS/hyperdrive-trigger
 
 A key feature of the Workflow Manager are triggers, which define when an ingestion should be executed and how it should be requested. The workflow manager supports cron-based triggers as well as triggers that listen to a notification topic.
+
+
+## How to build
+- Scala 2.12 (default)
+```
+mvn clean install
+```
+- Scala 2.11
+```
+mvn scala-cross-build:change-version -Pscala-2.11
+mvn clean install -Pscala-2.11
+```
+
+### E2E tests with Docker
+E2E tests require a running Docker instance on the executing machine and are not executed by default.
+To execute them, build using the profile `all-tests`
+```
+mvn clean test -Pall-tests
+```

api/pom.xml

@@ -17,12 +17,12 @@
 
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
-        <artifactId>parent-conf</artifactId>
+        <artifactId>parent-conf_2.12</artifactId>
         <groupId>za.co.absa.hyperdrive</groupId>
         <version>4.1.1-SNAPSHOT</version>
         <relativePath>../parent-conf/pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
-    <artifactId>api</artifactId>
+    <artifactId>api_2.12</artifactId>
     <packaging>jar</packaging>
 </project>

component-archetype/README.md

@@ -25,7 +25,7 @@ This is a Maven archetype for creating custom Hyperdrive components.
 
 Download the artifact to your local maven repository
 ```
-mvn dependency:get -Dartifact=za.co.absa.hyperdrive:component-archetype:4.0.0 
+mvn dependency:get -Dartifact=za.co.absa.hyperdrive:component-archetype:4.2.0 
 ```
 
 Update the local archetype catalog
@@ -37,8 +37,8 @@ Generate a skeleton project by executing the following command
 ```
 mvn archetype:generate \
     -DarchetypeGroupId=za.co.absa.hyperdrive \
-    -DarchetypeArtifactId=component-archetype \
-    -DarchetypeVersion=4.0.0 \
+    -DarchetypeArtifactId=component-archetype_2.12 \
+    -DarchetypeVersion=4.2.0 \
     -DgroupId=<groupId> \
     -DartifactId=<artifactId> \
     -Dversion=<artifact-version> 
@@ -47,6 +47,8 @@ mvn archetype:generate \
 - `<artifactId>` is the name for your artifact, e.g. mytransformer,
 - `<artifact-version>` is the version number of the artifact, e.g. 0.1.0-SNAPSHOT
 
+Hint: For Scala 2.11, use `-DarchetypeArtifactId=component-archetype_2.11`.
+
 ## Implementing a Hyperdrive component
 
 There are three types of Hyperdrive components: Reader, Transformer and Writer.

component-archetype/pom.xml

@@ -16,14 +16,14 @@
 
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
-        <artifactId>parent-conf</artifactId>
+        <artifactId>parent-conf_2.12</artifactId>
         <groupId>za.co.absa.hyperdrive</groupId>
         <version>4.1.1-SNAPSHOT</version>
         <relativePath>../parent-conf/pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
-    <artifactId>component-archetype</artifactId>
+    <artifactId>component-archetype_2.12</artifactId>
     <packaging>maven-archetype</packaging>
 
     <properties>

component-archetype/src/main/resources/archetype-resources/pom.xml

@@ -49,7 +49,7 @@
         <!-- Hyperdrive -->
         <dependency>
             <groupId>za.co.absa.hyperdrive</groupId>
-            <artifactId>api</artifactId>
+            <artifactId>api_\${scala.compat.version}</artifactId>
             <version>\${hyperdrive.version}</version>
             <scope>provided</scope>
         </dependency>

component-scanner/pom.xml

@@ -16,20 +16,20 @@
 
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
-        <artifactId>parent-conf</artifactId>
+        <artifactId>parent-conf_2.12</artifactId>
         <groupId>za.co.absa.hyperdrive</groupId>
         <version>4.1.1-SNAPSHOT</version>
         <relativePath>../parent-conf/pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
-    <artifactId>component-scanner</artifactId>
+    <artifactId>component-scanner_2.12</artifactId>
     <packaging>jar</packaging>
 
 
     <dependencies>
         <dependency>
             <groupId>za.co.absa.hyperdrive</groupId>
-            <artifactId>api</artifactId>
+            <artifactId>api_${scala.compat.version}</artifactId>
             <version>4.1.1-SNAPSHOT</version>
         </dependency>
 

driver/pom.xml

@@ -16,30 +16,30 @@
 
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
-        <artifactId>parent-conf</artifactId>
+        <artifactId>parent-conf_2.12</artifactId>
         <groupId>za.co.absa.hyperdrive</groupId>
         <version>4.1.1-SNAPSHOT</version>
         <relativePath>../parent-conf/pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
-    <artifactId>driver</artifactId>
+    <artifactId>driver_2.12</artifactId>
 
     <dependencies>
         <!--Components from the project-->
         <dependency>
             <groupId>za.co.absa.hyperdrive</groupId>
-            <artifactId>api</artifactId>
+            <artifactId>api_${scala.compat.version}</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>
             <groupId>za.co.absa.hyperdrive</groupId>
-            <artifactId>ingestor-default</artifactId>
+            <artifactId>ingestor-default_${scala.compat.version}</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>
             <groupId>za.co.absa.hyperdrive</groupId>
-            <artifactId>shared</artifactId>
+            <artifactId>shared_${scala.compat.version}</artifactId>
             <version>${project.version}</version>
         </dependency>
 

hyperdrive-release/pom.xml

@@ -16,19 +16,19 @@
 
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
-        <artifactId>parent-conf</artifactId>
+        <artifactId>parent-conf_2.12</artifactId>
         <groupId>za.co.absa.hyperdrive</groupId>
         <version>4.1.1-SNAPSHOT</version>
         <relativePath>../parent-conf/pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
-    <artifactId>hyperdrive-release</artifactId>
+    <artifactId>hyperdrive-release_2.12</artifactId>
 
     <dependencies>
         <dependency>
             <groupId>za.co.absa.hyperdrive</groupId>
-            <artifactId>driver</artifactId>
+            <artifactId>driver_${scala.compat.version}</artifactId>
             <version>${project.version}</version>
         </dependency>
     </dependencies>

ingestor-default/pom.xml

@@ -16,25 +16,25 @@
 
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
-        <artifactId>parent-conf</artifactId>
+        <artifactId>parent-conf_2.12</artifactId>
         <groupId>za.co.absa.hyperdrive</groupId>
         <version>4.1.1-SNAPSHOT</version>
         <relativePath>../parent-conf/pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
-    <artifactId>ingestor-default</artifactId>
+    <artifactId>ingestor-default_2.12</artifactId>
 
     <dependencies>
         <!--Hyperdrive-->
         <dependency>
             <groupId>za.co.absa.hyperdrive</groupId>
-            <artifactId>api</artifactId>
+            <artifactId>api_${scala.compat.version}</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>
             <groupId>za.co.absa.hyperdrive</groupId>
-            <artifactId>shared</artifactId>
+            <artifactId>shared_${scala.compat.version}</artifactId>
             <version>${project.version}</version>
         </dependency>