Various and assorted refactoring

* Use backoff decorators for exception handling
* Eliminate recursion as a retry mechanism
* Clean up log statements

Author: kocolosk

Dockerfile

@@ -2,7 +2,7 @@ FROM python:alpine
 
 MAINTAINER Adam Kocoloski [email protected]
 
-RUN pip install requests dnspython
+RUN pip install requests dnspython backoff
 
 COPY mem3_helper.py /opt/mem3_helper/
 

mem3_helper.py

@@ -10,51 +10,46 @@
 import time
 import dns.resolver
 import socket
+import backoff
 
-def discover_peers_dns(retries=5):
+def construct_service_record():
     # Drop our Pod's unique identity and replace with '_couchdb._tcp'
-    # to form the name of the SRV record
-    resource = '.'.join(['_couchdb', '_tcp'] + socket.getfqdn().split('.')[1:])
-    print ('SRV record', resource)
-    if retries > 0:
-        try:
-            answers = dns.resolver.query(resource, 'SRV')
-        except dns.resolver.NXDOMAIN:
-            print('DNS query for SRV records failed with NXDOMAIN error')
-            time.sleep(5)
-            discover_peers_dns(retries - 1)
-        # Erlang requires that we drop the trailing period from the absolute DNS
-        # name to form the hostname used for the Erlang node. This feels hacky
-        # but not sure of a more official answer
-        hostnames = [rdata.target.to_text()[:-1] for rdata in answers]
-        print(hostnames)
-        connect_the_dots(hostnames)
-    else:
-        print('Could not resolve resource ', resource)
+    return '.'.join(['_couchdb', '_tcp'] + socket.getfqdn().split('.')[1:])
+
+@backoff.on_exception(
+    backoff.expo,
+    dns.resolver.NXDOMAIN,
+    max_tries=10
+)
+def discover_peers(service_record):
+    print ('Resolving SRV record', service_record)
+    answers = dns.resolver.query(service_record, 'SRV')
+    # Erlang requires that we drop the trailing period from the absolute DNS
+    # name to form the hostname used for the Erlang node. This feels hacky
+    # but not sure of a more official answer
+    return [rdata.target.to_text()[:-1] for rdata in answers]
 
-def connect_the_dots(names, retries=5):
-    if retries > 0:
-        for name in names:
-            uri = "http://127.0.0.1:5986/_nodes/couchdb@{0}".format(name)
-            doc = {}
-            try:
-                resp = requests.put(uri, data=json.dumps(doc))
-                if resp.status_code == 404:
-                    print('CouchDB nodes DB does not exist yet')
-                    time.sleep(5)
-                    connect_the_dots(names, retries - 1)
-                print(name, resp.status_code)
-            except requests.exceptions.ConnectionError:
-                print('CouchDB admin port not up yet')
-                time.sleep(5)
-                connect_the_dots(names, retries - 1)
-        sleep_forever()
-    else:
-        print('Could not connect to local admin port to supply node names')
+@backoff.on_exception(
+    backoff.expo,
+    requests.exceptions.ConnectionError,
+    max_tries=10
+)
+def connect_the_dots(names):
+    for name in names:
+        uri = "http://127.0.0.1:5986/_nodes/couchdb@{0}".format(name)
+        doc = {}
+        resp = requests.put(uri, data=json.dumps(doc))
+        while resp.status_code == 404:
+            print('Waiting for _nodes DB to be created ...')
+            time.sleep(5)
+            resp = requests.put(uri, data=json.dumps(doc))
+        print('Adding cluster member', name, resp.status_code)
 
 def sleep_forever():
     while True:
         time.sleep(5)
 
 if __name__ == '__main__':
-    discover_peers_dns()
+    connect_the_dots(discover_peers(construct_service_record()))
+    print('Cluster membership populated!')
+    sleep_forever()