search: Use SQLAlchemy as a search-index backend

This avoids storing search-index state in the local process (as the
previous Python-dict-based implementation did).  Multiple registry
processes (e.g. gunicorn workers) can now use the same search-index
database and stay in sync with each other.

Author: wking

README.md

@@ -46,6 +46,7 @@ in action in the example below...
 
 common:
     loglevel: info
+    search_index: "_env:SEARCH_INDEX:"
 
 prod:
     loglevel: warn
@@ -204,6 +205,22 @@ dev:
   gs_secure: false
 ```
 
+### Search-engine options
+
+The Docker Registry can optionally use [SQLAlchemy][] to index
+repository information in a database, for the `GET /v1/search`
+[endpoint][search-endpoint].  You can configure the backend with a
+configuration like:
+
+```yaml
+search_index: "_env:SEARCH_INDEX:"
+```
+
+you can use the `SEARCH_INDEX` environment variable to configure the
+string passed to [create_engine][].  If the configured `search_index`
+is empty, no index is built, and the search endpoint always returns
+empty results.
+
 ### Email options
 
 Settings these options makes the Registry send an email on each code Exception:
@@ -404,3 +421,9 @@ pip install tox
 cd docker-registry/
 tox
 ```
+
+[SQLAlchemy]: http://docs.sqlalchemy.org/
+[search-endpoint]:
+  http://docs.docker.io/en/latest/reference/api/index_api/#get--v1-search
+[create_engine]:
+  http://docs.sqlalchemy.org/en/latest/core/engines.html#sqlalchemy.create_engine

config/config_sample.yml

@@ -31,6 +31,8 @@ common:
     # Let gunicorn set this environment variable or set a random string here
     secret_key: _env:SECRET_KEY
 
+    search_index: "_env:SEARCH_INDEX:"
+
 
 # This is the default configuration when no flavor is specified
 dev:

config/config_test.yml

@@ -14,3 +14,5 @@ test:
     gs_access_key: _env:GS_ACCESS_KEY
     gs_secret_key: _env:GS_SECRET_KEY 
     gs_secure: false
+
+    search_index: sqlite:////tmp/docker-registry.db

lib/storage/__init__.py

@@ -12,15 +12,13 @@ class Storage(object):
     """Storage is organized as follow:
     $ROOT/images/<image_id>/json
     $ROOT/images/<image_id>/layer
-    $ROOT/index/json
     $ROOT/repositories/<namespace>/<repository_name>/<tag_name>
     """
 
     # Useful if we want to change those locations later without rewriting
     # the code which uses Storage
     repositories = 'repositories'
     images = 'images'
-    index = 'index'
     # Set the IO buffer to 128kB
     buffer_size = 128 * 1024
     # By default no storage plugin supports it
@@ -87,9 +85,6 @@ def private_flag_path(self, namespace, repository):
     def is_private(self, namespace, repository):
         return self.exists(self.private_flag_path(namespace, repository))
 
-    def index_path(self):
-        return '{0}/json'.format(self.index)
-
     def get_content(self, path):
         raise NotImplementedError
 

registry/search.py

@@ -1,20 +1,56 @@
 __all__ = ['get_search']
 
 import flask
-import simplejson as json
-
+try:
+    import sqlalchemy
+    import sqlalchemy.exc
+    import sqlalchemy.ext.declarative
+    import sqlalchemy.orm
+    import sqlalchemy.sql.functions
+except ImportError as e:
+    _sqlalchemy_import_error = e
+    sqlalchemy = None
+
+import config
 import signals
 import storage
 import toolkit
 
 from .app import app
 
 
+cfg = config.load()
+index = None
 store = storage.load()
-#index = Index()
 
 
-class Index (dict):
+if sqlalchemy:
+    Base = sqlalchemy.ext.declarative.declarative_base()
+
+    class Version (Base):
+        "Schema version for the search-index database"
+        __tablename__ = 'version'
+
+        id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
+
+        def __repr__(self):
+            return '<{0}(id={1})>'.format(type(self).__name__, self.id)
+
+    class Repository (Base):
+        "Repository description"
+        __tablename__ = 'repository'
+
+        id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
+        name = sqlalchemy.Column(
+            sqlalchemy.String, nullable=False, unique=True)
+        description = sqlalchemy.Column(sqlalchemy.String)
+
+        def __repr__(self):
+            return "<{0}(name='{1}', description='{2}')>".format(
+                type(self).__name__, self.name, self.description)
+
+
+class Index (object):
     """Maintain an index of repository data
 
     The index is a dictionary.  The keys are
@@ -23,43 +59,33 @@ class Index (dict):
 
       index['library/ubuntu'] = 'An ubuntu image...'
     """
-    def __init__(self):
-        super(Index, self).__init__()
+    def __init__(self, database):
+        self._engine = sqlalchemy.create_engine(database)
+        self._session = sqlalchemy.orm.sessionmaker(bind=self._engine)
         self.version = 1
-        self.load()
+        self._setup_database()
         signals.repository_created.connect(self._handler_repository_created)
-        signals.repository_updated.connect(self._handler_repository_created)
+        signals.repository_updated.connect(self._handler_repository_updated)
         signals.repository_deleted.connect(self._handler_repository_deleted)
 
-    def load(self):
-        regenerated = False
+    def _setup_database(self):
+        session = self._session()
         try:
-            index_content = store.get_content(store.index_path())
-        except (OSError, IOError):
-            index_data = self._regenerate_index()
-            regenerated = True
-        else:
-            data = json.loads(index_content)
-            if data['version'] != self.version:
+            version = session.query(
+                sqlalchemy.sql.functions.max(Version.id)).first()[0]
+        except sqlalchemy.exc.OperationalError:
+            version = None
+        if version:
+            if version != self.version:
                 raise NotImplementedError(
-                    'unrecognized search index version {0}'.format(
-                        data['version']))
-            index_data = data['index']
-        self.clear()
-        self.update(index_data)
-        if regenerated:
-            self.save()
-
-    def save(self):
-        index_data = {
-            'version': self.version,
-            'index': dict(self),
-        }
-        store.put_content(store.index_path(), json.dumps(index_data))
-
-    def _regenerate_index(self):
-        index_data = {}
-        description = ''  # TODO(wking): store descriptions
+                    'unrecognized search index version {0}'.format(version))
+        else:
+            self._generate_index(session=session)
+        session.close()
+
+    def _generate_index(self, session):
+        Base.metadata.create_all(self._engine)
+        session.add(Version(id=self.version))
         try:
             namespace_paths = list(
                 store.list_directory(path=store.repositories))
@@ -74,41 +100,68 @@ def _regenerate_index(self):
                 repository_paths = []
             for path in repository_paths:
                 repository = path.rsplit('/', 1)[-1]
-                key = '{0}/{1}'.format(namespace, repository)
-                index_data[key] = description
-        return index_data
+                name = '{0}/{1}'.format(namespace, repository)
+                description = None  # TODO(wking): store descriptions
+                session.add(Repository(name=name, description=description))
+        session.commit()
 
     def _handler_repository_created(
             self, sender, namespace, repository, value):
-        key = '{0}/{1}'.format(namespace, repository)
+        name = '{0}/{1}'.format(namespace, repository)
         description = ''  # TODO(wking): store descriptions
-        self[key] = description
-        self.save()
-
-    def _handler_repository_deleted(self, sender, namespace, repository):
-        key = '{0}/{1}'.format(namespace, repository)
-        try:
-            self.pop(key)
-        except KeyError:
-            pass
-        else:
-            self.save()
+        session = self._session()
+        session.add(Repository(name=name, description=description))
+        session.commit()
+        session.close()
 
+    def _handler_repository_updated(
+            self, sender, namespace, repository, value):
+        name = '{0}/{1}'.format(namespace, repository)
+        description = ''  # TODO(wking): store descriptions
+        session = self._session()
+        session.query(Repository).filter(
+            Repository.name == name).update(
+                values={'description': description},
+                synchronize_session=False)
+        session.commit()
+        session.close()
 
-index = Index()
+    def _handler_repository_deleted(self, sender, namespace, repository):
+        name = '{0}/{1}'.format(namespace, repository)
+        session = self._session()
+        session.query(Repository).filter(Repository.name == name).delete()
+        session.commit()
+        session.close()
+
+    def results(self, search_term):
+        session = self._session()
+        like_term = '%{}%'.format(search_term)
+        repositories = session.query(Repository).filter(
+            sqlalchemy.sql.or_(
+                Repository.name.like(like_term),
+                Repository.description.like(like_term)))
+        return [
+            {
+                'name': repo.name,
+                'description': repo.description,
+            }
+            for repo in repositories]
+
+
+# Enable the search index
+if cfg.search_index:
+    if not sqlalchemy:
+        raise _sqlalchemy_import_error
+    index = Index(database=cfg.search_index)
 
 
 @app.route('/v1/search', methods=['GET'])
 def get_search():
     search_term = flask.request.args.get('q', '')
-    results = [
-        {
-            'name': name,
-            'description': description,
-        }
-        for name, description in index.items()
-        if search_term in name
-        or search_term in description]
+    if index is None:
+        results = []
+    else:
+        results = index.results(search_term=search_term)
     return toolkit.response({
         'query': search_term,
         'num_results': len(results),

requirements.txt

@@ -12,5 +12,6 @@ python-keystoneclient==0.3.1
 python-swiftclient==1.8.0
 redis==2.8.0
 rsa==3.1.2
+sqlalchemy==0.9.2
 bugsnag==1.3.1
 gcs-oauth2-boto-plugin>=1.3