minor fix and upgrade to 1.3.1

Author: Yang Lei

README.md

@@ -45,7 +45,10 @@ com.cloudant.spark.CloudantPartitionedPrunedFilteredRP|path|PrunedFilteredScan|
 
 ### Verified on Spark level
 
-1.3.0
+Spark Version | Release #
+--- | ---
+1.3.0 | v0.1
+1.3.1 | v1.3.1.1
 
 ## Sample application 
 
@@ -189,6 +192,11 @@ path||riak: search index name; cloudant: as database name if database does not p
 		at com.esotericsoftware.kryo.serializers.FieldSerializer.read(FieldSerializer.java:217)
 		at com.esotericsoftware.kryo.Kryo.readClassAndObject(Kryo.java:732)
 
+* Parallel loading relation provider hit issue on mesos cluster:
+	com.typesafe.config.ConfigException$Missing: No configuration setting found for key 'spray'
+	at com.typesafe.config.impl.SimpleConfig.findKey(SimpleConfig.java:124)
+	at com.typesafe.config.impl.SimpleConfig.find(SimpleConfig.java:147)
+	
 * Schema is calculated on the first document w/o any predicate push down. Need a better approach
 
 * Cloudant search index query does not support "paging" through skip and limit. Push down may not get the best performance anyway. Will try changes + bulk read api next.

cloudant-spark-sql/build.sbt

@@ -14,7 +14,7 @@ resolvers ++= Seq(
 )
 
 libraryDependencies ++= {
-  val sparkV =  "1.3.0"
+  val sparkV =  "1.3.1"
   val sprayV = "1.3.2"
   val playJsonV = "2.2.3"
   Seq(

cloudant-spark-sql/project/assembly.sbt

@@ -13,5 +13,3 @@ spark-sql {
   		minInPartition = 10
   	}
 }
-  		
-

cloudant-spark-sql/project/build.properties

@@ -24,6 +24,7 @@ import scala.util.control.Breaks._
 import play.api.libs.json.JsUndefined
 import java.net.URLEncoder
 import com.cloudant.spark.common._
+import play.api.libs.json.JsNumber
 
 /**
  * @author yanglei
@@ -33,9 +34,19 @@ import com.cloudant.spark.common._
     private lazy val dbUrl = {"http://"+ host+"/"+dbName}
 
     val pkField = "_id"
+    val defaultIndex = "_all_docs" // "_changes" does not work for partition
 
     override def getPostUrl(): String ={dbUrl}
-    
+    override def getLastUrl(skip: Int): String = {
+      if (skip ==0 ) null
+      else s"$dbUrl/$defaultIndex?limit=$skip"
+    }
+    override def getLastNum(result: JsValue): JsValue = {result \ "last_seq"}
+    override def getTotalUrl(url: String) = {
+      if (url.contains('?')) url+"&limit=1"
+      else  url+"?limit=1"
+    }
+
     def getOneUrl(): String = { dbUrl+ "/_all_docs?limit=1&include_docs=true"}
 
     def getRangeUrl(field: String = null, start: Any = null, startInclusive:Boolean = false, end:Any =null, endInclusive: Boolean =false,includeDoc: Boolean = true): (String, Boolean) = {
@@ -74,17 +85,25 @@ import com.cloudant.spark.common._
           val condition = calculateCondition(field, start, startInclusive, end, endInclusive)
           (dbUrl+"/"+indexName+"?q="+condition, true)
       }else
-        (dbUrl + "/_all_docs" ,false)
+        (s"$dbUrl/$defaultIndex" ,false)
     }
 
-    def getSubSetUrl (url: String, skip: Int, limit: Int) : String ={
-      val suffix = "include_docs=true&limit="+limit+"&skip="+skip
+    def getSubSetUrl (url: String, skip: Int, limit: Int)(implicit convertSkip:(Int) => String): String ={
+      val suffix = {
+        if (url.indexOf("_all_docs")>0) "include_docs=true&limit="+limit+"&skip="+skip
+        else if (url.indexOf("_changes")>0) "include_docs=true&limit="+limit+"&since="+convertSkip(skip)
+        else "include_docs=true&limit="+limit // TODO Index query does not support subset query. Should disable Partitioned loading?
+      }
       if (url.indexOf('?')>0) url+"&"+suffix
       else url+"?"+suffix
     }
 
-    def getTotalRows(result: JsValue): JsValue = {
-        result \ "total_rows"
+    def getTotalRows(result: JsValue): Int = {
+        val value = result \ "total_rows"
+        value match {
+            case s : JsUndefined => (result \ "pending").as[JsNumber].value.intValue() +1
+            case _ =>  value.as[JsNumber].value.intValue()
+          }
     }
 
     def getRows(result: JsValue): Seq[JsValue] = {