Merge pull request #553 from cloudant/534-add-seq-interval-in-changes-feed

534 add sequence interval to changes

Author: emlaver

CHANGES.md

@@ -4,6 +4,8 @@
   [Bluemix documentation](https://console.bluemix.net/docs/services/Cloudant/guides/iam.html#ibm-cloud-identity-and-access-management)
   for more details.
 - [IMPROVED] Updated documentation by replacing deprecated links with the latest Bluemix or CouchDB links.
+- [IMPROVED] Added `seq_interval` to improve `Changes` API throughput when replicating from
+             a CouchDB 2.x endpoint.
 
 # 2.0.2 (2017-06-20)
 - [FIXED] Removed cloudant-sync-datastore-android project dependency

cloudant-sync-datastore-core/src/main/java/com/cloudant/sync/internal/mazha/CouchClient.java

@@ -284,10 +284,6 @@ private Map<String, Object> getDefaultChangeFeedOptions() {
         return options;
     }
 
-    public ChangesResult changes(Object since) {
-        return this.changes(since, null);
-    }
-
     public ChangesResult changes(Object since, Integer limit) {
         return this.changes(null, null, since, limit);
     }
@@ -306,6 +302,12 @@ public ChangesResult changes(String filterName, Map<String, String> filterParame
         if (limit != null) {
             options.put("limit", limit);
         }
+        // seq_interval: improve performance and reduce load on the remote database
+        if(limit != null) {
+            options.put("seq_interval", limit);
+        } else {
+            options.put("seq_interval", 1000);
+        }
         return this.changes(options);
     }
 

cloudant-sync-datastore-core/src/test/java/com/cloudant/sync/internal/mazha/ChangesFeedTest.java

@@ -22,6 +22,8 @@
 
 import com.cloudant.common.RequireRunningCouchDB;
 
+import java.io.IOException;
+import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -40,7 +42,7 @@ public void changes_dbChangesMustSuccessfullyReturn() {
         Object lastSeq = "0";
 
         { // Two docs changed
-            ChangesResult changes = client.changes(lastSeq);
+            ChangesResult changes = getChangesSince(lastSeq);
             Map<String, List<String>> changedRevIds = findChangedRevisionIds(changes);
 
             Assert.assertThat(changedRevIds.size(), is(equalTo(2)));
@@ -54,7 +56,7 @@ public void changes_dbChangesMustSuccessfullyReturn() {
         Response res3 = ClientTestUtils.createHelloWorldDoc(client);
 
         { // One doc changed
-            ChangesResult changes = client.changes(lastSeq);
+            ChangesResult changes = getChangesSince(lastSeq);
             Map<String, List<String>> changedRevIds = findChangedRevisionIds(changes);
 
             Assert.assertThat(changedRevIds.size(), is(equalTo(1)));
@@ -65,7 +67,7 @@ public void changes_dbChangesMustSuccessfullyReturn() {
         }
 
         { // No changes
-            ChangesResult changes = client.changes(lastSeq);
+            ChangesResult changes = getChangesSince(lastSeq);
             Assert.assertTrue(changes.size() == 0);
         }
     }
@@ -85,21 +87,52 @@ public Map<String, List<String>> findChangedRevisionIds(ChangesResult changesRes
     @Test(expected = NoResourceException.class)
     public void changes_dbNotExist_exception() {
         client.deleteDb();
-        client.changes("1");
+        getChangesSince("1");
     }
 
     @Test
     public void changes_docWithConflicts_conflictsShouldBeReturned() {
         // Not sure how to changed conflicts yet
     }
 
+    @Test
+    public void changes_dbChangesMustSuccessfullyReturnWithSeqInterval() throws IOException,
+            URISyntaxException {
+        org.junit.Assume.assumeTrue(ClientTestUtils.isCouchDBV2(client.getRootUri()));
+        Response res1 = ClientTestUtils.createHelloWorldDoc(client);
+        Response res2 = ClientTestUtils.createHelloWorldDoc(client);
+        ClientTestUtils.createHelloWorldDoc(client);
+        Object lastSeq = "0";
+
+        {
+            // Use batch interval and seq_interval of 4
+            ChangesResult changes = client.changes(lastSeq, 4);
+            Map<String, List<String>> changedRevIds = findChangedRevisionIds(changes);
+
+            Assert.assertThat(changedRevIds.size(), is(equalTo(3)));
+            Assert.assertThat(changedRevIds.keySet(), hasItems(res1.getId(), res2.getId()));
+            Assert.assertThat(changedRevIds.get(res1.getId()), hasItem(res1.getRev()));
+            Assert.assertThat(changedRevIds.get(res2.getId()), hasItem(res2.getRev()));
+            // last two shouldn't have seq
+            Assert.assertNull(changes.getResults().get(1).getSeq());
+            Assert.assertNull(changes.getResults().get(2).getSeq());
+
+            lastSeq = changes.getLastSeq();
+        }
+
+        { // No changes
+            ChangesResult changes = getChangesSince(lastSeq);
+            Assert.assertTrue(changes.size() == 0);
+        }
+    }
+
     @Test
     public void changes_dbWithConflicts_changesMustSuccessfullyReturn() {
         ClientTestUtils.createHelloWorldDoc(client);
         Object lastSeq = "0";
 
         {
-            ChangesResult changes = client.changes(lastSeq);
+            ChangesResult changes = getChangesSince(lastSeq);
             Assert.assertEquals(1, changes.size());
             lastSeq = changes.getLastSeq();
         }
@@ -109,7 +142,7 @@ public void changes_dbWithConflicts_changesMustSuccessfullyReturn() {
         Response res2 = ClientTestUtils.createHelloWorldDoc(client);
 
         {
-            ChangesResult changes = client.changes(lastSeq);
+            ChangesResult changes = getChangesSince(lastSeq);
             Assert.assertEquals(2, changes.size());
 
             List<String> doc1ChangedRevs = findChangesRevs(changes, res1.getId());
@@ -122,7 +155,7 @@ public void changes_dbWithConflicts_changesMustSuccessfullyReturn() {
         }
     }
 
-    public List<String> findChangesRevs(ChangesResult changes, String id) {
+    private List<String> findChangesRevs(ChangesResult changes, String id) {
         List<String> changedRevs = new ArrayList<String>();
         for(ChangesResult.Row row : changes.getResults()) {
             if(row.getId().equals(id)) {
@@ -133,4 +166,8 @@ public List<String> findChangesRevs(ChangesResult changes, String id) {
         }
         return changedRevs;
     }
+
+    private ChangesResult getChangesSince(Object since) {
+        return client.changes(since, null);
+    }
 }

cloudant-sync-datastore-core/src/test/java/com/cloudant/sync/internal/mazha/ClientTestUtils.java

@@ -31,9 +31,13 @@
 import org.apache.commons.io.IOUtils;
 import org.junit.Assert;
 
+import java.io.BufferedReader;
+import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
 import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -235,5 +239,12 @@ public static List<String> getRemoteRevisionIDs(URI uri, CouchConfig config) thr
         return revisions;
     }
 
+    public static boolean isCouchDBV2(URI uri) throws URISyntaxException, IOException {
+        URI root = new URI(uri.getScheme() + "://" + uri.getAuthority());
+        HttpConnection connection = Http.GET(root);
+        String response = connection.execute().responseAsString();
+        return response.contains("\"version\":\"2.");
+    }
+
 
 }