Update DataLake example by removing implicit calls

Author: GezimSejdiu

sansa-examples-spark/src/main/resources/datalake/config

@@ -11,43 +11,44 @@
 		"entity": "Person"
 	}
 	,	{
-		"type": "csv",
+		"type": "parquet",
 		"options": {
-			"header": "true",
-			"delimiter": ",",
-			"mode": "DROPMALFORMED"
+			"spark_sql_parquet_filterPushdown": "true"
 		},
-		"source": "src/main/resources/Data/review.csv",
+		"source": "src/main/resources/Data/review.parquet",
 		"entity": "Review"
 	}
 	,	{
-		"type": "csv",
+		"type": "mongodb",
 		"options": {
-			"header": "true",
-			"delimiter": ",",
-			"mode": "DROPMALFORMED"
+			"url": "127.0.0.1",
+			"database": "bsbm",
+			"collection": "offer",
+			"options": ""
 		},
-		"source": "src/main/resources/Data/offer.csv",
+		"source": "//Offer",
 		"entity": "Offer"
 	}
 	,	{
-		"type": "csv",
+		"type": "cassandra",
 		"options": {
-			"header": "true",
-			"delimiter": ",",
-			"mode": "DROPMALFORMED"
+			"keyspace": "db",
+			"table": "product"
 		},
-		"source": "src/main/resources/Data/product.csv",
+		"source": "//Product",
 		"entity": "Product"
 	}
 	,	{
-		"type": "csv",
+		"type": "jdbc",
 		"options": {
-			"header": "true",
-			"delimiter": ",",
-			"mode": "DROPMALFORMED"
+			"url": "jdbc:mysql://localhost:3306/benchmark?useUnicode=true&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=UTC&autoReconnect=true&useSSL=false",
+
+			"driver": "com.mysql.cj.jdbc.Driver",
+			"dbtable": "producer",
+			"user": "root",
+			"password": "root"
 		},
-		"source": "src/main/resources/Data/producer.csv",
+		"source": "//Producer",
 		"entity": "Producer"
 	}
 	],

sansa-examples-spark/src/main/resources/datalake/mappings.ttl

@@ -15,8 +15,8 @@
 
 <#ProducerMapping>
 	rml:logicalSource [
-		rml:source "src/main/resources/Data/producer.csv";
-		nosql:store nosql:csv
+		rml:source "//Producer";
+		nosql:store nosql:jdbc
 	];
 	rr:subjectMap [
 		rr:template "http://example.com/{nr}";
@@ -55,8 +55,8 @@
 
 <#ReviewMapping>
 	rml:logicalSource [
-		rml:source "src/main/resources/Data/review.csv";
-		nosql:store nosql:csv
+		rml:source "src/main/resources/Data/review.parquet";
+		nosql:store nosql:parquet
 	];
 	rr:subjectMap [
 		rr:template "http://example.com/{nr}";
@@ -165,8 +165,8 @@
 
 <#OfferMapping>
 	rml:logicalSource [
-		rml:source "src/main/resources/Data/offer.csv";
-		nosql:store nosql:csv
+		rml:source "//Offer";
+		nosql:store nosql:mongodb
 	];
 	rr:subjectMap [
 		rr:template "http://example.com/{_id}";
@@ -225,8 +225,8 @@
 
 <#ProductMapping>
 	rml:logicalSource [
-		rml:source "src/main/resources/Data/product.csv";
-		nosql:store nosql:csv
+		rml:source "//Product";
+		nosql:store nosql:cassandra
 	];
 	rr:subjectMap [
 		rr:template "http://example.com/{nr}";
@@ -316,4 +316,4 @@
 	rr:predicateObjectMap [
 		rr:predicate bsbm:productPropertyTextual2;
 		rr:objectMap [rml:reference "propertyTex2"]
-	].
+	].

sansa-examples-spark/src/main/scala/net/sansa_stack/examples/spark/query/DataLake.scala

@@ -2,6 +2,7 @@ package net.sansa_stack.examples.spark.query
 
 import scala.collection.mutable
 
+import net.sansa_stack.query.spark.datalake.DataLakeEngine
 import net.sansa_stack.query.spark.query._
 import org.apache.jena.riot.Lang
 import org.apache.spark.sql.SparkSession
@@ -31,7 +32,8 @@ object DataLake {
       .master("local[*]")
       .getOrCreate()
 
-    val result = spark.sparqlDL(queryFile, mappingsFile, configFile)
+    // val result = spark.sparqlDL(queryFile, mappingsFile, configFile)
+    val result = DataLakeEngine.run(queryFile, mappingsFile, configFile, spark)
     result.show()
 
     spark.stop
@@ -40,8 +42,8 @@ object DataLake {
 
   case class Config(
     queryFile: String = getClass.getResource("/datalake/queries/Q1.sparql").getPath,
-    mappingsFile: String = getClass.getResource("/datalake/config_csv-only").getPath,
-    configFile: String = getClass.getResource("/datalake/mappings_csv-only.ttl").getPath)
+    mappingsFile: String = getClass.getResource("/datalake/config").getPath,
+    configFile: String = getClass.getResource("/datalake/mappings.ttl").getPath)
 
   val parser = new scopt.OptionParser[Config]("Sparqlify example") {