Merge branch 'dev'

Author: tariqdaouda

CHANGELOG.rst

@@ -1,3 +1,16 @@
+1.3.5
+=====
+
+* restoreIndex and restoreIndexes in collection will restore previously deleted indexes
+* added max_conflict_retries to handle arango's 1200
+* added single session so AikidoSessio.Holders can share a single request session
+* added task deletion to tests reset
+* added drop() to tasks to remove all tasks in one command 
+* better documentation of connection class
+* False is not considered a Null value while validating
+* Removed redundant document creation functions
+* More explicit validation error with field name
+
 1.3.4
 =====
 * Bugfix: Query iterator now returns all elements instead of a premature empty list

DESCRIPTION.rst

@@ -1,5 +1,5 @@
 Python Object Wrapper for ArangoDB_ with built-in validation
-===========================================================
+=============================================================
 
 pyArango aims to be an easy to use driver for ArangoDB with built in validation. Collections are treated as types that apply to the documents within. You can be 100% permissive or enforce schemas and validate fields on set, on save or on both.
 
@@ -15,4 +15,4 @@ pyArango is developed by `Tariq Daouda`_, the full source code is available from
 For the latest news about pyArango, you can follow me on twitter `@tariqdaouda`_.
 If you have any issues with it, please file a github issue.
 
-.. _@tariqdaouda: https://www.twitter.com/tariqdaouda
+.. _@tariqdaouda: https://www.twitter.com/tariqdaouda

pyArango/collection.py

@@ -315,25 +315,11 @@ def delete(self):
             raise DeletionError(data["errorMessage"], data)
 
     def createDocument(self, initDict = None):
-        """create and returns a document populated with the defaults or with the values in initDict"""
-        if initDict is not None:
-            return self.createDocument_(initDict)
-        else:
-            return self.createDocument_(self.defaultDocument)
-            # if self._validation["on_load"]:
-            #     self._validation["on_load"] = False
-                # self._validation["on_load"] = True
-                # return self.createDocument_(self.defaultDocument)
-
-    def createDocument_(self, initDict = None):
         """create and returns a completely empty document or one populated with initDict"""
         res = dict(self.defaultDocument)
-        if initDict is None:
-            initV = {}
-        else:
-            initV = initDict
-        res.update(initV)
-
+        if initDict is not None:
+            res.update(initDict)
+ 
         return self.documentClass(self, res)
 
     def _writeBatch(self):
@@ -614,6 +600,34 @@ def ensureFulltextIndex(self, fields, minLength = None, name = None):
             self.indexes_by_name[name] = ind
         return ind
 
+    def ensureIndex(self, index_type, fields, name=None, **index_args):
+        """Creates an index of any type."""
+        data = {
+            "type" : index_type,
+            "fields" : fields,
+        }
+        data.update(index_args)
+        
+        if name:
+            data["name"] = name
+
+        ind = Index(self, creationData = data)
+        self.indexes[index_type][ind.infos["id"]] = ind
+        if name:
+            self.indexes_by_name[name] = ind
+        return ind
+
+    def restoreIndexes(self, indexes_dct=None):
+        """restores all previously removed indexes"""
+        if indexes_dct is None:
+            indexes_dct = self.indexes
+
+        for typ in indexes_dct.keys():
+            if typ != "primary":
+                for name, idx in indexes_dct[typ].items():
+                    infos = dict(idx.infos)
+                    del infos["fields"]
+                    self.ensureIndex(typ, idx.infos["fields"], **infos)
 
     def validatePrivate(self, field, value):
         """validate a private field value"""
@@ -757,8 +771,10 @@ def bulkSave(self, docs, onDuplicate="error", **params):
         if (r.status_code == 201) and "error" not in data:
             return True
         else:
-            if data["errors"] > 0:
+            if "errors" in data and data["errors"] > 0:
                 raise UpdateError("%d documents could not be created" % data["errors"], data)
+            elif data["error"]:
+                raise UpdateError("Documents could not be created", data)
 
         return data["updated"] + data["created"]
 
@@ -909,15 +925,9 @@ def validateField(cls, fieldName, value):
                 raise e
         return valValue
 
-    def createEdge(self):
+    def createEdge(self, initValues = None):
         """Create an edge populated with defaults"""
-        return self.createDocument()
-
-    def createEdge_(self, initValues = None):
-        """Create an edge populated with initValues"""
-        if not initValues:
-            initValues = {}
-        return self.createDocument_(initValues)
+        return self.createDocument(initValues)
 
     def getInEdges(self, vertex, rawResults = False):
         """An alias for getEdges() that returns only the in Edges"""

pyArango/connection.py

@@ -33,9 +33,10 @@ class AikidoSession(object):
     """
 
     class Holder(object):
-        def __init__(self, fct, auth, verify=True):
+        def __init__(self, fct, auth, max_conflict_retries=5, verify=True):
             self.fct = fct
             self.auth = auth
+            self.max_conflict_retries = max_conflict_retries
             if not isinstance(verify, bool) and not isinstance(verify, CA_Certificate) and not not isinstance(verify, str) :
                 raise ValueError("'verify' argument can only be of type: bool, CA_Certificate or str ")
             self.verify = verify
@@ -49,7 +50,12 @@ def __call__(self, *args, **kwargs):
                 kwargs["verify"] = self.verify
 
             try:
-                ret = self.fct(*args, **kwargs)
+                status_code = 1200
+                retry = 0
+                while status_code == 1200 and retry < self.max_conflict_retries :
+                    ret = self.fct(*args, **kwargs)
+                    status_code = ret.status_code
+                    retry += 1
             except:
                 print ("===\nUnable to establish connection, perhaps arango is not running.\n===")
                 raise
@@ -62,7 +68,7 @@ def __call__(self, *args, **kwargs):
             ret.json = JsonHook(ret)
             return ret
 
-    def __init__(self, username, password, verify=True, max_retries=5, log_requests=False):
+    def __init__(self, username, password, verify=True, max_conflict_retries=5, max_retries=5, single_session=True, log_requests=False):
         if username:
             self.auth = (username, password)
         else:
@@ -71,19 +77,33 @@ def __init__(self, username, password, verify=True, max_retries=5, log_requests=
         self.verify = verify
         self.max_retries = max_retries
         self.log_requests = log_requests
+        self.max_conflict_retries = max_conflict_retries
+
+        self.session = None
+        if single_session:
+            self.session = self._make_session()
 
         if log_requests:
             self.log = {}
             self.log["nb_request"] = 0
             self.log["requests"] = {}
 
+    def _make_session(self):
+        session = requests.Session()
+        http = requests.adapters.HTTPAdapter(max_retries=self.max_retries)
+        https = requests.adapters.HTTPAdapter(max_retries=self.max_retries)
+        session.mount('http://', http)
+        session.mount('https://', https)
+
+        return session
+
     def __getattr__(self, request_function_name):
+        if self.session is not None:
+            session = self.session
+        else :
+            session = self._make_session()
+
         try:
-            session = requests.Session()
-            http = requests.adapters.HTTPAdapter(max_retries=self.max_retries)
-            https = requests.adapters.HTTPAdapter(max_retries=self.max_retries)
-            session.mount('http://', http)
-            session.mount('https://', https)
             request_function = getattr(session, request_function_name)
         except AttributeError:
             raise AttributeError("Attribute '%s' not found (no Aikido move available)" % request_function_name)
@@ -95,15 +115,46 @@ def __getattr__(self, request_function_name):
             log["nb_request"] += 1
             log["requests"][request_function.__name__] += 1
 
-        return AikidoSession.Holder(request_function, auth, verify)
+        return AikidoSession.Holder(request_function, auth, max_conflict_retries=self.max_conflict_retries, verify=verify)
 
     def disconnect(self):
         pass
 
 class Connection(object):
     """This is the entry point in pyArango and directly handles databases.
     @param arangoURL: can be either a string url or a list of string urls to different coordinators
-    @param use_grequests: allows for running concurent requets."""
+    @param use_grequests: allows for running concurent requets.
+
+    Parameters
+    ----------
+    arangoURL: list or str
+        list of urls or url for connecting to the db
+
+    username: str
+        for credentials
+    password: str
+        for credentials
+    verify: bool
+        check the validity of the CA certificate
+    verbose: bool
+        flag for addictional prints during run
+    statsdClient: instance
+        statsd instance    
+    reportFileName: str
+        where to save statsd report
+    loadBalancing: str
+        type of load balancing between collections
+    use_grequests: bool
+        parallelise requests using gevents. Use with care as gevents monkey patches python, this could have unintended concequences on other packages
+    use_jwt_authentication: bool
+        use JWT authentication
+    use_lock_for_reseting_jwt: bool
+        use lock for reseting gevents authentication
+    max_retries: int
+        max number of retries for a request
+    max_conflict_retries: int
+        max number of requests for a conflict error (1200 arangodb error). Does not work with gevents (grequests),
+    """
 
     LOAD_BLANCING_METHODS = {'round-robin', 'random'}
 
@@ -120,6 +171,7 @@ def __init__(self,
         use_jwt_authentication=False,
         use_lock_for_reseting_jwt=True,
         max_retries=5,
+        max_conflict_retries=5
     ):
 
         if loadBalancing not in Connection.LOAD_BLANCING_METHODS:
@@ -132,6 +184,7 @@ def __init__(self,
         self.use_jwt_authentication = use_jwt_authentication
         self.use_lock_for_reseting_jwt = use_lock_for_reseting_jwt
         self.max_retries = max_retries
+        self.max_conflict_retries = max_conflict_retries
         self.action = ConnectionAction(self)
 
         self.databases = {}
@@ -211,7 +264,16 @@ def resetSession(self, username=None, password=None, verify=True):
                 verify
             )
         else:
-            self.session = AikidoSession(username, password, verify, self.max_retries)
+            # self.session = AikidoSession(username, password, verify, self.max_retries)
+            self.session = AikidoSession(
+                username=username,
+                password=password,
+                verify=verify,
+                single_session=True,
+                max_conflict_retries=self.max_conflict_retries,
+                max_retries=self.max_retries,
+                log_requests=False
+            )
 
     def reload(self):
         """Reloads the database list.

pyArango/database.py

@@ -128,10 +128,10 @@ def createCollection(self, className = 'Collection', **colProperties):
             colProperties["type"] = CONST.COLLECTION_DOCUMENT_TYPE
 
         payload = json.dumps(colProperties, default=str)
-        r = self.connection.session.post(self.getCollectionsURL(), data = payload)
-        data = r.json()
+        req = self.connection.session.post(self.getCollectionsURL(), data = payload)
+        data = req.json()
 
-        if r.status_code == 200 and not data["error"]:
+        if req.status_code == 200 and not data["error"]:
             col = colClass(self, data)
             self.collections[col.name] = col
             return self.collections[col.name]
@@ -210,6 +210,10 @@ def hasGraph(self, name):
         """returns true if the databse has a graph by the name of 'name'"""
         return name in self.graphs
 
+    def __contains__(self, name):
+        """if name in database"""
+        return self.hasCollection(name) or self.hasGraph(name)
+
     def dropAllCollections(self):
         """drops all public collections (graphs included) from the database"""
         for graph_name in self.graphs:
@@ -335,15 +339,15 @@ def fetch_list(
             batch_index = 0
             result = []
             while True:
-                if len(query.response['result']) is 0:
+                if len(query.response['result']) == 0:
                     break
                 result.extend(query.response['result'])
                 batch_index += 1
                 query.nextBatch()
         except StopIteration:
             if log is not None:
                 log(result)
-            if len(result) is not 0:
+            if len(result) != 0:
                 return result
         except:
             raise
@@ -401,7 +405,7 @@ def fetch_list_as_batches(
             )
             batch_index = 0
             while True:
-                if len(query.response['result']) is 0:
+                if len(query.response['result']) == 0:
                     break
                 if log is not None:
                     log(
@@ -456,7 +460,7 @@ def no_fetch_run(
         ).response
         if log is not None:
             log(response["result"])
-        if len(response["result"]) is 0:
+        if len(response["result"]) == 0:
             return
         raise AQLFetchError("No results should be returned for the query.")
 

pyArango/doc/source/index.rst

@@ -9,7 +9,7 @@ Welcome to pyArango's documentation!
 .. image:: https://travis-ci.org/tariqdaouda/pyArango.svg?branch=1.2.2
     :target: https://travis-ci.org/tariqdaouda/pyArango
 .. image:: https://img.shields.io/badge/python-2.7%2C%203.5-blue.svg
-.. image:: https://img.shields.io/badge/arangodb-3.0-blue.svg
+.. image:: https://img.shields.io/badge/arangodb-3.6-blue.svg
 
 pyArango is a python driver for the NoSQL amazing database ArangoDB_ first written by `Tariq Daouda`_. As of January 2019 pyArango was handed over to the ArangoDB-Community that now ensures the developement and maintenance. It has a very light interface and built in validation. pyArango is distributed under the ApacheV2 Licence and the full source code can be found on github_.
 

pyArango/document.py

@@ -70,7 +70,10 @@ def validateField(self, field):
                 return self[field].validate()
 
             if field in self.patchStore:
-                return self.validators[field].validate(self.patchStore[field])
+                try:
+                    return self.validators[field].validate(self.patchStore[field])
+                except ValidationError as e:
+                    raise ValidationError( "'%s' -> %s" % ( field, str(e)) )
             else:
                 try:
                     return self.validators[field].validate(self.store[field])

pyArango/graph.py

@@ -171,7 +171,7 @@ def createEdge(self, collectionName, _fromId, _toId, edgeAttributes, waitForSync
         raise CreationError("Unable to create edge, %s" % r.json()["errorMessage"], data)
 
     def link(self, definition, doc1, doc2, edgeAttributes, waitForSync = False):
-        "A shorthand for createEdge that takes two documents as input"
+        """A shorthand for createEdge that takes two documents as input"""
         if type(doc1) is DOC.Document:
             if not doc1._id:
                 doc1.save()
@@ -189,7 +189,7 @@ def link(self, definition, doc1, doc2, edgeAttributes, waitForSync = False):
         return self.createEdge(definition, doc1_id, doc2_id, edgeAttributes, waitForSync)
 
     def unlink(self, definition, doc1, doc2):
-        "deletes all links between doc1 and doc2"
+        """deletes all links between doc1 and doc2"""
         links = self.database[definition].fetchByExample( {"_from": doc1._id,"_to" : doc2._id}, batchSize = 100)
         for l in links:
             self.deleteEdge(l)