Merge pull request #165 from yukim/CDM-84

[CDM-84] Improve usage of Apache Spark

Author: msmygit

pom.xml

@@ -46,11 +46,13 @@
           <artifactId>log4j-slf4j-impl</artifactId>
         </exclusion>
       </exclusions>
+      <scope>provided</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-sql_${scala.main.version}</artifactId>
       <version>${spark.version}</version>
+      <scope>provided</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
@@ -66,6 +68,7 @@
           <artifactId>apache-log4j-extras</artifactId>
         </exclusion>
       </exclusions>
+      <scope>provided</scope>
     </dependency>
     <dependency>
       <groupId>com.datastax.spark</groupId>

src/main/scala/com/datastax/cdm/job/BaseJob.scala

@@ -82,7 +82,7 @@ abstract class BaseJob[T: ClassTag] extends App {
     this.parts = getParts(numSplits)
     this.slices = sContext.parallelize(parts.asScala.toSeq, parts.size);
     abstractLogger.info("PARAM Calculated -- Total Partitions: " + parts.size())
-    abstractLogger.info("Spark parallelize created : " + slices.count() + " slices!");
+    abstractLogger.info("Spark parallelize created : " + slices.getNumPartitions + " slices!");
   }
 
   def getParts(pieces: Int): util.Collection[T]