Merge pull request #305 from cloudant/304-fix-bad-startkey-in-all-doc-iter

304 fix bad startkey in all doc iter

Author: smithsz

CHANGES.rst

@@ -6,6 +6,7 @@
 - [FIXED] Fixed Cloudant exception code 409 with 412 when creating a database that already exists.
 - [FIXED] Catch error if ``throw_on_exists`` flag is ``False`` for document create.
 - [FIXED] Fixed /_all_docs call where ``keys`` is an empty list.
+- [FIXED] Issue where docs with IDs that sorted lower than 0 were not returned when iterating through _all_docs.
 
 2.4.0 (2017-02-14)
 ==================

src/cloudant/_2to3.py

@@ -32,6 +32,9 @@
 # pylint: disable=undefined-variable
 LONGTYPE = long if PY2 else int
 
+# pylint: disable=undefined-variable
+UNICHR = unichr if PY2 else chr
+
 if PY2:
     # pylint: disable=wrong-import-position,no-name-in-module,import-error,unused-import
     from urllib import quote as url_quote, quote_plus as url_quote_plus

src/cloudant/database.py

@@ -630,18 +630,20 @@ def __iter__(self, remote=True):
         if not remote:
             super(CouchDatabase, self).__iter__()
         else:
-            next_startkey = '0'
+            # Use unicode Null U+0000 as the initial lower bound to ensure any
+            # document id could exist in the results set.
+            next_startkey = u'\u0000'
             while next_startkey is not None:
                 docs = self.all_docs(
-                    limit=self._fetch_limit + 1,  # Get one extra doc
-                                                  # to use as
-                                                  # next_startkey
+                    limit=self._fetch_limit,
                     include_docs=True,
                     startkey=next_startkey
                 ).get('rows', [])
 
-                if len(docs) > self._fetch_limit:
-                    next_startkey = docs.pop()['id']
+                if len(docs) >= self._fetch_limit:
+                    # Ensure the next document batch contains ids that sort
+                    # strictly higher than the previous document id fetched.
+                    next_startkey = docs[-1]['id'] + u'\u0000'
                 else:
                     # This is the last batch of docs, so we set
                     # ourselves up to break out of the while loop

tests/unit/database_tests.py

@@ -29,6 +29,7 @@
 import os
 import uuid
 
+from cloudant._2to3 import UNICHR
 from cloudant.result import Result, QueryResult
 from cloudant.error import CloudantArgumentError, CloudantDatabaseException
 from cloudant.document import Document
@@ -605,6 +606,39 @@ def test_document_iteration_over_fetch_limit(self):
             self.assertEqual(doc['name'], 'julia')
             self.assertEqual(doc['age'], int(id[len(id) - 3: len(id)]))
 
+    def test_document_iteration_completeness(self):
+        """
+        Test __iter__ works as expected, fetching all documents from the
+        database.
+        """
+        for _ in self.db:
+            self.fail('There should be no documents in the database yet!!')
+
+        # sample code point ranges
+        include_ranges = [
+            (0x0023, 0x0026),
+            (0x00A1, 0x00AC),
+            (0x0370, 0x0377),
+            (0x037A, 0x037E),
+            (0x0384, 0x038A),
+            (0x16A0, 0x16F0),
+            (0x2C60, 0x2C7F)
+        ]
+
+        all_docs = [{'_id': UNICHR(i) + UNICHR(j)} for a, b in include_ranges
+                                                   for i in range(a, b)
+                                                   for j in range(a, b)]
+        batch_size = 500
+        for i in range(0, len(all_docs), batch_size):
+            self.db.bulk_docs(all_docs[i:i+batch_size])
+
+        doc_count = 0
+        for i, doc in enumerate(self.db):
+            doc_count += 1
+            self.assertEqual(doc['_id'], all_docs[i]['_id'])
+
+        self.assertEqual(doc_count, len(all_docs))
+
     def test_document_iteration_returns_valid_documents(self):
         """
         This test will check that the __iter__ method returns documents that are