Merge pull request #416 from dotcloud/272-dont-die-on-redis-errors

Don't crash on Redis connection errors

Author: dmp42

depends/docker-registry-core/docker_registry/core/lru.py

@@ -65,7 +65,11 @@ def wrapper(*args):
         content = args[-1]
         key = args[-2]
         key = cache_key(key)
-        redis_conn.set(key, content)
+        try:
+            redis_conn.set(key, content)
+        except redis.exceptions.ConnectionError as e:
+            logging.warning("LRU: Redis connection error: {0}".format(e))
+
         return f(*args)
     if redis_conn is None:
         return f
@@ -77,13 +81,21 @@ def get(f):
     def wrapper(*args):
         key = args[-1]
         key = cache_key(key)
-        content = redis_conn.get(key)
+        try:
+            content = redis_conn.get(key)
+        except redis.exceptions.ConnectionError as e:
+            logging.warning("LRU: Redis connection error: {0}".format(e))
+            content = None
+
         if content is not None:
             return content
         # Refresh cache
         content = f(*args)
         if content is not None:
-            redis_conn.set(key, content)
+            try:
+                redis_conn.set(key, content)
+            except redis.exceptions.ConnectionError as e:
+                logging.warning("LRU: Redis connection error: {0}".format(e))
         return content
     if redis_conn is None:
         return f
@@ -95,7 +107,10 @@ def remove(f):
     def wrapper(*args):
         key = args[-1]
         key = cache_key(key)
-        redis_conn.delete(key)
+        try:
+            redis_conn.delete(key)
+        except redis.exceptions.ConnectionError as e:
+            logging.warning("LRU: Redis connection error: {0}".format(e))
         return f(*args)
     if redis_conn is None:
         return f

docker_registry/images.py

@@ -289,8 +289,7 @@ def put_image_checksum(image_id):
     # Checksum is ok, we remove the marker
     store.remove(mark_path)
     # We trigger a task on the diff worker if it's running
-    if cache.redis_conn:
-        layers.diff_queue.push(image_id)
+    layers.enqueue_diff(image_id)
     return toolkit.response()
 
 

docker_registry/lib/layers.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 
+import logging
 import tarfile
 import tempfile
 
@@ -28,10 +29,20 @@
     tarfile.GNUTYPE_SPARSE: 'S',
 }
 
+logger = logging.getLogger(__name__)
+
 # queue for requesting diff calculations from workers
 diff_queue = rqueue.CappedCollection(cache.redis_conn, "diff-worker", 1024)
 
 
+def enqueue_diff(image_id):
+    try:
+        if cache.redis_conn:
+            diff_queue.push(image_id)
+    except cache.redis.exceptions.ConnectionError as e:
+        logger.warning("Diff queue: Redis connection error: {0}".format(e))
+
+
 def generate_ancestry(image_id, parent_id=None):
     if not parent_id:
         store.put_content(store.image_ancestry_path(image_id),

docker_registry/lib/mirroring.py

@@ -105,9 +105,16 @@ def wrapper(namespace, repository, *args, **kwargs):
             # client GETs a single tag
             tag_path = store.tag_path(namespace, repository, kwargs['tag'])
 
-        data = cache.redis_conn.get('{0}:{1}'.format(
-            cache.cache_prefix, tag_path
-        ))
+        try:
+            data = cache.redis_conn.get('{0}:{1}'.format(
+                cache.cache_prefix, tag_path
+            ))
+        except cache.redis.exceptions.ConnectionError as e:
+            data = None
+            logger.warning("Diff queue: Redis connection error: {0}".format(
+                e
+            ))
+
         if data is not None:
             return toolkit.response(data=data, raw=True)
         source_resp = lookup_source(
@@ -117,9 +124,15 @@ def wrapper(namespace, repository, *args, **kwargs):
             return resp
         data = source_resp.content
         headers = _response_headers(source_resp.headers)
-        cache.redis_conn.setex('{0}:{1}'.format(
-            cache.cache_prefix, tag_path
-        ), tags_cache_ttl, data)
+        try:
+            cache.redis_conn.setex('{0}:{1}'.format(
+                cache.cache_prefix, tag_path
+            ), tags_cache_ttl, data)
+        except cache.redis.exceptions.ConnectionError as e:
+            logger.warning("Diff queue: Redis connection error: {0}".format(
+                e
+            ))
+
         return toolkit.response(data=data, headers=headers,
                                 raw=True)
     return wrapper