Merge pull request #1 from techmonad/version_update

Update spark and elasticsearch version

Author: satendrakumar

build.sbt

@@ -2,13 +2,13 @@ name := "spark-data-pipeline"
 
 version := "1.0"
 
-scalaVersion := "2.11.11"
+scalaVersion := "2.11.12"
 
 
 libraryDependencies ++= Seq(
-  "org.apache.spark" %% "spark-core" % "2.3.1",
+  "org.apache.spark" %% "spark-core" % "2.4.0",
   "com.univocity" % "univocity-parsers" % "2.6.4",
-  "org.elasticsearch" %% "elasticsearch-spark-20" % "6.3.0",
+  "org.elasticsearch" %% "elasticsearch-spark-20" % "6.6.0",
   "edu.stanford.nlp" % "stanford-corenlp" % "3.6.0" artifacts(Artifact("stanford-corenlp", "models"), Artifact("stanford-corenlp")),
   "ch.qos.logback" % "logback-classic" % "1.2.3",
   "org.json4s" %% "json4s-native" % "3.5.4",

src/main/scala/com/techmonad/pipeline/reader/CSVReader.scala

@@ -10,13 +10,14 @@ object CSVReader extends TryHelper {
 
   def read(url: String, delimiter: Char = ',')(implicit sc: SparkContext): RDD[Record] = {
     val headers = sc.textFile(url).first()
-    sc.textFile(url).mapPartitionsWithIndex {
-      case (index, itr) =>
-        if (index == 0)
-          readFile(itr.drop(1), headers)
-        else
-          readFile(itr, headers)
-    }
+    sc.textFile(url)
+      .mapPartitionsWithIndex {
+        case (index, itr) =>
+          if (index == 0)
+            readFile(itr.drop(1), headers)
+          else
+            readFile(itr, headers)
+      }
   }