Merge pull request #444 from dotcloud/config-enhance

Config rehaul

Author: dmp42

README.md

@@ -252,7 +252,7 @@ common:
   mirroring:
     source: https://registry-1.docker.io
     source_index: https://index.docker.io
-    tags_cache_ttl: 864000 # 10 days
+    tags_cache_ttl: 172800 # 2 days
 ```
 
 ### Cache options

config/config_mirror.yml

@@ -1,12 +1,13 @@
 # The `common' part is automatically included (and possibly overriden by all
 # other flavors)
-common:
+common: &common
     mirroring:
         source: https://registry-1.docker.io
         source_index: https://index.docker.io
-        tags_cache_ttl: 864000  # seconds
+        tags_cache_ttl: 172800  # seconds
 
 dev:
+    <<: *common
     storage: local
     storage_path: /tmp/registry
     loglevel: debug

config/config_sample.yml

@@ -1,31 +1,42 @@
-# The `common' part is automatically included (and possibly overriden by all
-# other flavors)
-common:
-    loglevel: _env:LOGLEVEL:debug
+# All other flavors inherit the `common' config snippet
+common: &common
+    # Default log level is info
+    loglevel: _env:LOGLEVEL:info
+    # By default, the registry acts standalone (eg: doesn't query the index)
+    standalone: _env:STANDALONE:true
+    # The default endpoint to use (if NOT standalone) is index.docker.io
+    index_endpoint: _env:INDEX_ENDPOINT:https://index.docker.io
+    # Storage redirect is disabled
     storage_redirect: _env:STORAGE_REDIRECT
-    standalone: true
-    index_endpoint: _env:INDEX_ENDPOINT
+    # Token auth is enabled (if NOT standalone)
     disable_token_auth: _env:DISABLE_TOKEN_AUTH
+    # No priv key
     privileged_key: _env:PRIVILEGED_KEY
-
+    # No search backend
     search_backend: _env:SEARCH_BACKEND
+    # SQLite search backend
     sqlalchemy_index_database: _env:SQLALCHEMY_INDEX_DATABASE:sqlite:////tmp/docker-registry.db
 
+    # Mirroring is not enabled
     mirroring:
         source: _env:MIRROR_SOURCE # https://registry-1.docker.io
         source_index: _env:MIRROR_SOURCE_INDEX # https://index.docker.io
-        tags_cache_ttl: _env:MIRROR_TAGS_CACHE_TTL # 864000 # seconds
+        tags_cache_ttl: _env:MIRROR_TAGS_CACHE_TTL:172800 # seconds
+
+    # cache:
+    #     host: _env:CACHE_REDIS_HOST:localhost
+    #     port: _env:CACHE_REDIS_PORT:6379
+    #     db: 0
+    #     password: _env:CACHE_REDIS_PASSWORD
 
-    # Enabling LRU cache for small files. This speeds up read/write on small files
+    # Enabling LRU cache for small files
+    # This speeds up read/write on small files
     # when using a remote storage backend (like S3).
-    cache:
-        host: _env:CACHE_REDIS_HOST
-        port: _env:CACHE_REDIS_PORT
-        password: _env:CACHE_REDIS_PASSWORD
-    cache_lru:
-        host: _env:CACHE_LRU_REDIS_HOST
-        port: _env:CACHE_LRU_REDIS_PORT
-        password: _env:CACHE_LRU_REDIS_PASSWORD
+    # cache_lru:
+    #     host: _env:CACHE_LRU_REDIS_HOST:localhost
+    #     port: _env:CACHE_LRU_REDIS_PORT:6379
+    #     db: 0
+    #     password: _env:CACHE_LRU_REDIS_PASSWORD
 
     # Enabling these options makes the Registry send an email on each code Exception
     email_exceptions:
@@ -40,14 +51,14 @@ common:
     # Enable bugsnag (set the API key)
     bugsnag: _env:BUGSNAG
 
-
-
 local: &local
+    <<: *common
     storage: local
     storage_path: _env:STORAGE_PATH:/tmp/registry
 
 
 s3: &s3
+    <<: *common
     storage: s3
     s3_region: _env:AWS_REGION
     s3_bucket: _env:AWS_BUCKET
@@ -63,6 +74,7 @@ s3: &s3
 # https://developers.google.com/storage/docs/reference/v1/getting-startedv1#keys
 # for details on access and secret keys.
 gcs:
+    <<: *common
     storage: gcs
     boto_bucket: _env:GCS_BUCKET
     storage_path: _env:STORAGE_PATH:/registry
@@ -79,6 +91,7 @@ gcs:
 
 # This flavor is for storing images in Openstack Swift
 swift: &swift
+    <<: *common
     storage: swift
     storage_path: _env:STORAGE_PATH:/registry
     # keystone authorization
@@ -92,8 +105,9 @@ swift: &swift
 # This flavor stores the images in Glance (to integrate with openstack)
 # See also: https://github.com/dotcloud/openstack-docker
 glance: &glance
+    <<: *common
     storage: glance
-    storage_alternate: _env:GLANCE_STORAGE_ALTERNATE:local
+    storage_alternate: _env:GLANCE_STORAGE_ALTERNATE:file
     storage_path: _env:STORAGE_PATH:/tmp/registry
 
 openstack:
@@ -110,6 +124,7 @@ openstack-swift:
     <<: *glance-swift
 
 elliptics:
+    <<: *common
     storage: elliptics
     elliptics_nodes: _env:ELLIPTICS_NODES
     elliptics_wait_timeout: _env:ELLIPTICS_WAIT_TIMEOUT:60
@@ -127,13 +142,15 @@ elliptics:
 # This is the default configuration when no flavor is specified
 dev: &dev
     <<: *local
+    loglevel: _env:LOGLEVEL:debug
     search_backend: _env:SEARCH_BACKEND:sqlalchemy
 
-# This flavor is automatically used by unit tests
+# This flavor is used by unit tests
 test:
     <<: *dev
+    index_endpoint: https://indexstaging-docker.dotcloud.com
     standalone: True
-    storage_path: _env:STORAGE_PATH:~/tmp/test
+    storage_path: _env:STORAGE_PATH:./tmp/test
 
 # To specify another flavor, set the environment variable SETTINGS_FLAVOR
 # $ export SETTINGS_FLAVOR=prod

depends/docker-registry-core/docker_registry/core/__init__.py

@@ -39,7 +39,7 @@
 __credits__ = []
 
 __license__ = 'Apache 2.0'
-__version__ = '1.0.7'
+__version__ = '2.0.0'
 __maintainer__ = 'Docker'
 __email__ = '[email protected]'
 __status__ = 'Production'

depends/docker-registry-core/docker_registry/core/boto.py

@@ -129,7 +129,7 @@ class Base(driver.Base):
 
     def __init__(self, path=None, config=None):
         self._config = config
-        self._root_path = config.get('storage_path', '/test')
+        self._root_path = path or '/test'
         self._boto_conn = self.makeConnection()
         self._boto_bucket = self._boto_conn.get_bucket(
             self._config.boto_bucket)

depends/docker-registry-core/docker_registry/testing/utils.py

@@ -42,8 +42,9 @@ def __repr__(self):
         return repr(self._config)
 
     def __getattr__(self, key):
-        if key in self._config:
-            return self._config[key]
+        if key not in self._config:
+            return None
+        return self._config[key]
 
-    def get(self, *args, **kwargs):
-        return self._config.get(*args, **kwargs)
+    def __getitem__(self, key):
+        return getattr(self, key)

docker_registry/app.py

@@ -18,9 +18,8 @@
 
 app = flask.Flask('docker-registry')
 cfg = config.load()
-loglevel = getattr(logging, cfg.get('loglevel', 'INFO').upper())
 logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s',
-                    level=loglevel)
+                    level=getattr(logging, cfg.loglevel.upper()))
 
 
 @app.route('/_ping')
@@ -48,20 +47,20 @@ def after_request(response):
 def init():
     # Configure the email exceptions
     info = cfg.email_exceptions
-    if info and 'smtp_host' in info:
-        mailhost = info['smtp_host']
-        mailport = info.get('smtp_port')
+    if info and info.smtp_host:
+        mailhost = info.smtp_host
+        mailport = info.smtp_port
         if mailport:
             mailhost = (mailhost, mailport)
-        smtp_secure = info.get('smtp_secure', None)
+        smtp_secure = info.smtp_secure
         secure_args = _adapt_smtp_secure(smtp_secure)
         mail_handler = logging.handlers.SMTPHandler(
             mailhost=mailhost,
-            fromaddr=info['from_addr'],
-            toaddrs=[info['to_addr']],
+            fromaddr=info.from_addr,
+            toaddrs=[info.to_addr],
             subject='Docker registry exception',
-            credentials=(info['smtp_login'],
-                         info['smtp_password']),
+            credentials=(info.smtp_login,
+                         info.smtp_password),
             secure=secure_args)
         mail_handler.setLevel(logging.ERROR)
         app.logger.addHandler(mail_handler)
@@ -91,9 +90,9 @@ def _adapt_smtp_secure(value):
     if isinstance(value, basestring):
         # a string - wrap it in the tuple
         return (value,)
-    if isinstance(value, dict):
+    if isinstance(value, config.Config):
         assert set(value.keys()) <= set(['keyfile', 'certfile'])
-        return (value['keyfile'], value.get('certfile', None))
+        return (value.keyfile, value.certfile)
     if value:
         return ()
 

docker_registry/lib/cache.py

@@ -7,67 +7,48 @@
 
 from . import config
 
-
-# Default options
+logger = logging.getLogger(__name__)
 
 redis_conn = None
 cache_prefix = None
 
+cfg = config.load()
+
 
 def init():
-    cfg = config.load()
-    enable_redis_cache(cfg)
-    enable_redis_lru(cfg)
+    enable_redis_cache(cfg.cache, cfg.storage_path)
+    enable_redis_lru(cfg.cache_lru, cfg.storage_path)
 
 
-def enable_redis_cache(cfg):
+def enable_redis_cache(cache, path):
     global redis_conn, cache_prefix
-    cache = cfg.cache
-    if not cache:
+    if not cache or not cache.host:
+        logger.warn('Cache storage disabled!')
         return
 
-    logging.info('Enabling storage cache on Redis')
-    if not isinstance(cache, dict):
-        cache = {}
-    redis_opts = {
-        'host': 'localhost',
-        'port': 6379,
-        'db': 0,
-        'password': None
-    }
-    for k, v in cache.iteritems():
-        redis_opts[k] = v
-    logging.info('Redis config: {0}'.format(redis_opts))
-    redis_conn = redis.StrictRedis(host=redis_opts['host'],
-                                   port=int(redis_opts['port']),
-                                   db=int(redis_opts['db']),
-                                   password=redis_opts['password'])
-    cache_prefix = 'cache_path:{0}'.format(cfg.get('storage_path', '/'))
+    logger.info('Enabling storage cache on Redis')
+    logger.info('Redis config: {0}'.format(cache))
+    redis_conn = redis.StrictRedis(
+        host=cache.host,
+        port=int(cache.port),
+        db=int(cache.db),
+        password=cache.password
+    )
+    cache_prefix = 'cache_path:{0}'.format(path or '/')
 
 
-def enable_redis_lru(cfg):
-    cache = cfg.cache_lru
-    if not cache:
+def enable_redis_lru(cache, path):
+    if not cache or not cache.host:
+        logger.warn('LRU cache disabled!')
         return
-    logging.info('Enabling lru cache on Redis')
-    if not isinstance(cache, dict):
-        cache = {}
-    redis_opts = {
-        'host': 'localhost',
-        'port': 6379,
-        'db': 0,
-        'password': None
-    }
-    for k, v in cache.iteritems():
-        redis_opts[k] = v
-
-    logging.info('Redis lru config: {0}'.format(redis_opts))
+    logger.info('Enabling lru cache on Redis')
+    logger.info('Redis lru config: {0}'.format(cache))
     lru.init(
-        host=redis_opts['host'],
-        port=int(redis_opts['port']),
-        db=int(redis_opts['db']),
-        password=redis_opts['password'],
-        path=cfg.get('storage_path', '/')
+        host=cache.host,
+        port=cache.port,
+        db=cache.db,
+        password=cache.password,
+        path=path or '/'
     )
 
 init()