Fixes to follow new Config

- replace get(key) and ['key'] calls by dotted notation
- removed defaults from the codebase (as they now all live in the config)
- tweak existence tests
- simplified calls to load() when possible
- assume proper typage

Docker-DCO-1.1-Signed-off-by: Mangled Deutz <[email protected]> (github: dmp42)

Author: Mangled Deutz

README.md

@@ -252,7 +252,7 @@ common:
   mirroring:
     source: https://registry-1.docker.io
     source_index: https://index.docker.io
-    tags_cache_ttl: 864000 # 10 days
+    tags_cache_ttl: 172800 # 2 days
 ```
 
 ### Cache options

config/config_mirror.yml

@@ -1,12 +1,13 @@
 # The `common' part is automatically included (and possibly overriden by all
 # other flavors)
-common:
+common: &common
     mirroring:
         source: https://registry-1.docker.io
         source_index: https://index.docker.io
-        tags_cache_ttl: 864000  # seconds
+        tags_cache_ttl: 172800  # seconds
 
 dev:
+    <<: *common
     storage: local
     storage_path: /tmp/registry
     loglevel: debug

config/config_sample.yml

@@ -21,7 +21,7 @@ common: &common
     mirroring:
         source: _env:MIRROR_SOURCE # https://registry-1.docker.io
         source_index: _env:MIRROR_SOURCE_INDEX # https://index.docker.io
-        tags_cache_ttl: _env:MIRROR_TAGS_CACHE_TTL # 864000 # seconds
+        tags_cache_ttl: _env:MIRROR_TAGS_CACHE_TTL:172800 # seconds
 
     cache:
         host: _env:CACHE_REDIS_HOST:localhost

docker_registry/app.py

@@ -18,9 +18,8 @@
 
 app = flask.Flask('docker-registry')
 cfg = config.load()
-loglevel = getattr(logging, cfg.get('loglevel', 'INFO').upper())
 logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s',
-                    level=loglevel)
+                    level=getattr(logging, cfg.loglevel.upper()))
 
 
 @app.route('/_ping')
@@ -48,20 +47,20 @@ def after_request(response):
 def init():
     # Configure the email exceptions
     info = cfg.email_exceptions
-    if info and 'smtp_host' in info:
-        mailhost = info['smtp_host']
-        mailport = info.get('smtp_port')
+    if info and info.smtp_host:
+        mailhost = info.smtp_host
+        mailport = info.smtp_port
         if mailport:
             mailhost = (mailhost, mailport)
-        smtp_secure = info.get('smtp_secure', None)
+        smtp_secure = info.smtp_secure
         secure_args = _adapt_smtp_secure(smtp_secure)
         mail_handler = logging.handlers.SMTPHandler(
             mailhost=mailhost,
-            fromaddr=info['from_addr'],
-            toaddrs=[info['to_addr']],
+            fromaddr=info.from_addr,
+            toaddrs=[info.to_addr],
             subject='Docker registry exception',
-            credentials=(info['smtp_login'],
-                         info['smtp_password']),
+            credentials=(info.smtp_login,
+                         info.smtp_password),
             secure=secure_args)
         mail_handler.setLevel(logging.ERROR)
         app.logger.addHandler(mail_handler)
@@ -91,9 +90,9 @@ def _adapt_smtp_secure(value):
     if isinstance(value, basestring):
         # a string - wrap it in the tuple
         return (value,)
-    if isinstance(value, dict):
+    if isinstance(value, config.Config):
         assert set(value.keys()) <= set(['keyfile', 'certfile'])
-        return (value['keyfile'], value.get('certfile', None))
+        return (value.keyfile, value.certfile)
     if value:
         return ()
 

docker_registry/lib/cache.py

@@ -7,67 +7,48 @@
 
 from . import config
 
-
-# Default options
+logger = logging.getLogger(__name__)
 
 redis_conn = None
 cache_prefix = None
 
+cfg = config.load()
+
 
 def init():
-    cfg = config.load()
-    enable_redis_cache(cfg)
-    enable_redis_lru(cfg)
+    enable_redis_cache(cfg.cache, cfg.storage_path)
+    enable_redis_lru(cfg.cache_lru, cfg.storage_path)
 
 
-def enable_redis_cache(cfg):
+def enable_redis_cache(cache, path):
     global redis_conn, cache_prefix
-    cache = cfg.cache
-    if not cache:
+    if not cache or not cache.host:
+        logger.warn('Cache storage disabled!')
         return
 
-    logging.info('Enabling storage cache on Redis')
-    if not isinstance(cache, dict):
-        cache = {}
-    redis_opts = {
-        'host': 'localhost',
-        'port': 6379,
-        'db': 0,
-        'password': None
-    }
-    for k, v in cache.iteritems():
-        redis_opts[k] = v
-    logging.info('Redis config: {0}'.format(redis_opts))
-    redis_conn = redis.StrictRedis(host=redis_opts['host'],
-                                   port=int(redis_opts['port']),
-                                   db=int(redis_opts['db']),
-                                   password=redis_opts['password'])
-    cache_prefix = 'cache_path:{0}'.format(cfg.get('storage_path', '/'))
+    logger.info('Enabling storage cache on Redis')
+    logger.info('Redis config: {0}'.format(cache))
+    redis_conn = redis.StrictRedis(
+        host=cache.host,
+        port=int(cache.port),
+        db=int(cache.db),
+        password=cache.password
+    )
+    cache_prefix = 'cache_path:{0}'.format(path or '/')
 
 
-def enable_redis_lru(cfg):
-    cache = cfg.cache_lru
-    if not cache:
+def enable_redis_lru(cache, path):
+    if not cache or not cache.host:
+        logger.warn('LRU cache disabled!')
         return
-    logging.info('Enabling lru cache on Redis')
-    if not isinstance(cache, dict):
-        cache = {}
-    redis_opts = {
-        'host': 'localhost',
-        'port': 6379,
-        'db': 0,
-        'password': None
-    }
-    for k, v in cache.iteritems():
-        redis_opts[k] = v
-
-    logging.info('Redis lru config: {0}'.format(redis_opts))
+    logger.info('Enabling lru cache on Redis')
+    logger.info('Redis lru config: {0}'.format(cache))
     lru.init(
-        host=redis_opts['host'],
-        port=int(redis_opts['port']),
-        db=int(redis_opts['db']),
-        password=redis_opts['password'],
-        path=cfg.get('storage_path', '/')
+        host=cache.host,
+        port=cache.port,
+        db=cache.db,
+        password=cache.password,
+        path=path or '/'
     )
 
 init()

docker_registry/lib/mirroring.py

@@ -10,14 +10,12 @@
 from . import cache
 from . import config
 
-
-DEFAULT_CACHE_TAGS_TTL = 48 * 3600
 logger = logging.getLogger(__name__)
+cfg = config.load()
 
 
 def is_mirror():
-    cfg = config.load()
-    return bool(cfg.get('mirroring', False))
+    return bool(cfg.mirroring and cfg.mirroring.source)
 
 
 def _response_headers(base):
@@ -34,11 +32,9 @@ def _response_headers(base):
 
 def lookup_source(path, stream=False, source=None):
     if not source:
-        cfg = config.load()
-        mirroring_cfg = cfg.mirroring
-        if not mirroring_cfg:
+        if not is_mirror():
             return
-        source = cfg.mirroring['source']
+        source = cfg.mirroring.source
     source_url = '{0}{1}'.format(source, path)
     headers = {}
     for k, v in flask.request.headers.iteritems():
@@ -67,14 +63,12 @@ def lookup_source(path, stream=False, source=None):
 def source_lookup_tag(f):
     @functools.wraps(f)
     def wrapper(namespace, repository, *args, **kwargs):
-        cfg = config.load()
         mirroring_cfg = cfg.mirroring
         resp = f(namespace, repository, *args, **kwargs)
-        if not mirroring_cfg:
+        if not is_mirror():
             return resp
-        source = mirroring_cfg['source']
-        tags_cache_ttl = mirroring_cfg.get('tags_cache_ttl',
-                                           DEFAULT_CACHE_TAGS_TTL)
+        source = mirroring_cfg.source
+        tags_cache_ttl = mirroring_cfg.tags_cache_ttl
 
         if resp.status_code != 404:
             logger.debug('Status code is not 404, no source '
@@ -129,14 +123,13 @@ def source_lookup(cache=False, stream=False, index_route=False):
     def decorator(f):
         @functools.wraps(f)
         def wrapper(*args, **kwargs):
-            cfg = config.load()
             mirroring_cfg = cfg.mirroring
             resp = f(*args, **kwargs)
-            if not mirroring_cfg:
+            if not is_mirror():
                 return resp
-            source = mirroring_cfg['source']
-            if index_route:
-                source = mirroring_cfg.get('source_index', source)
+            source = mirroring_cfg.source
+            if index_route and mirroring_cfg.source_index:
+                source = mirroring_cfg.source_index
             logger.debug('Source provided, registry acts as mirror')
             if resp.status_code != 404:
                 logger.debug('Status code is not 404, no source '

docker_registry/status.py

@@ -18,7 +18,7 @@
 from .lib import cache
 from .lib import config
 
-_config = config.load()
+cfg = config.load()
 
 
 def redis_status():
@@ -41,7 +41,7 @@ def redis_status():
 def storage_status():
     message = ''
     try:
-        _storage = storage.load(_config.storage)
+        _storage = storage.load(cfg.storage)
         key = toolkit.gen_random_string()
         value = toolkit.gen_random_string()
         _storage.put_content(key, value)

docker_registry/storage/__init__.py

@@ -34,7 +34,7 @@ def load(kind=None):
         return _storage[kind]
 
     _storage[kind] = engine.fetch(kind)(
-        path=cfg.get('storage_path'),
+        path=cfg.storage_path,
         config=cfg)
 
     return _storage[kind]

docker_registry/toolkit.py

@@ -19,7 +19,7 @@
 from . import storage
 from .lib import config
 
-
+cfg = config.load()
 logger = logging.getLogger(__name__)
 _re_docker_version = re.compile('docker/([^\s]+)')
 _re_authorization = re.compile(r'(\w+)[:=][\s"]?([^",]+)"?')
@@ -98,8 +98,7 @@ def response(data=None, code=200, headers=None, raw=False):
 
 
 def validate_parent_access(parent_id):
-    cfg = config.load()
-    if cfg.standalone is True:
+    if cfg.standalone:
         return True
     auth = _parse_auth_header()
     if not auth:
@@ -108,12 +107,8 @@ def validate_parent_access(parent_id):
     if len(full_repos_name) != 2:
         logger.debug('validate_parent: Invalid repository field')
         return False
-    index_endpoint = cfg.index_endpoint
-    if index_endpoint is None:
-        index_endpoint = 'https://index.docker.io'
-    index_endpoint = index_endpoint.strip('/')
     url = '{0}/v1/repositories/{1}/{2}/layer/{3}/access'.format(
-        index_endpoint, full_repos_name[0], full_repos_name[1], parent_id
+        cfg.index_endpoint, full_repos_name[0], full_repos_name[1], parent_id
     )
     headers = {'Authorization': flask.request.headers.get('authorization')}
     resp = requests.get(url, verify=True, headers=headers)
@@ -136,12 +131,7 @@ def validate_token(auth):
     if len(full_repos_name) != 2:
         logger.debug('validate_token: Invalid repository field')
         return False
-    cfg = config.load()
-    index_endpoint = cfg.index_endpoint
-    if index_endpoint is None:
-        index_endpoint = 'https://index.docker.io'
-    index_endpoint = index_endpoint.strip('/')
-    url = '{0}/v1/repositories/{1}/{2}/images'.format(index_endpoint,
+    url = '{0}/v1/repositories/{1}/{2}/images'.format(cfg.index_endpoint,
                                                       full_repos_name[0],
                                                       full_repos_name[1])
     headers = {'Authorization': flask.request.headers.get('authorization')}
@@ -190,7 +180,6 @@ def _parse_auth_header():
 
 def check_token(args):
     logger.debug('args = {0}'.format(args))
-    cfg = config.load()
     if cfg.disable_token_auth is True or cfg.standalone is True:
         return True
     auth = _parse_auth_header()
@@ -224,8 +213,7 @@ def check_token(args):
 
 
 def check_signature():
-    cfg = config.load()
-    if not cfg.get('privileged_key'):
+    if not cfg.privileged_key:
         return False
     headers = flask.request.headers
     signature = headers.get('X-Signature')
@@ -242,7 +230,7 @@ def check_signature():
                        ['{}:{}'.format(k, headers[k]) for k in header_keys])
     logger.debug('Signed message: {}'.format(message))
     try:
-        return rsa.verify(message, sigdata, cfg.get('privileged_key'))
+        return rsa.verify(message, sigdata, cfg.privileged_key)
     except rsa.VerificationError:
         return False
 
@@ -303,10 +291,8 @@ def get_repository():
     return (parts[0], parts[1])
 
 
-def get_endpoints(cfg=None):
-    if not cfg:
-        cfg = config.load()
-    registry_endpoints = cfg.registry_endpoints
+def get_endpoints(overcfg=None):
+    registry_endpoints = (overcfg or cfg).registry_endpoints
     if not registry_endpoints:
         #registry_endpoints = socket.gethostname()
         registry_endpoints = flask.request.environ['HTTP_HOST']

docker_registry/wsgi.py

@@ -10,7 +10,7 @@
 if __name__ == '__main__':
     # Bind to PORT if defined, otherwise default to 5000.
     host = env.source('REGISTRY_HOST')
-    port = int(env.source('REGISTRY_PORT'))
+    port = env.source('REGISTRY_PORT')
     app.debug = True
     app.run(host=host, port=port)
     # Or you can run: