Merge pull request #247 from wking/search-index

Flesh out repository deletion and add basic search indexing

Author: shin-

README.md

@@ -36,8 +36,8 @@ environment: `export SETTINGS_FLAVOR=dev`
 The default environment is `dev`.
 
 NOTE: it's possible to load environment variables from the config file
-with a simple syntax: _env:VARIABLENAME. Check this syntax in action
-in the example below...
+with a simple syntax: `_env:VARIABLENAME[:DEFAULT]`. Check this syntax
+in action in the example below...
 
 
 #### Example config
@@ -46,6 +46,9 @@ in the example below...
 
 common:
     loglevel: info
+    search_backend: "_env:SEARCH_BACKEND:"
+    sqlalchemy_index_database:
+        "_env:SQLALCHEMY_INDEX_DATABASE:sqlite:////tmp/docker-registry.db"
 
 prod:
     loglevel: warn
@@ -208,6 +211,36 @@ dev:
   gs_secure: false
 ```
 
+### Search-engine options
+
+The Docker Registry can optionally index repository information in a
+database for the `GET /v1/search` [endpoint][search-endpoint].  You
+can configure the backend with a configuration like:
+
+```yaml
+search_backend: "_env:SEARCH_BACKEND:"
+```
+
+The `search_backend` setting selects the search backend to use.  If
+`search_backend` is empty, no index is built, and the search endpoint
+always returns empty results.  Currently supported backends and their
+backend-specific configuration options are:
+
+* `sqlalchemy': Use [SQLAlchemy][].
+    * The backing database is selected with
+      `sqlalchemy_index_database`, which is passed through to
+      [create_engine][].
+
+If `search_backend` is neither empty nor one of the above backends, it
+should point to a module:
+
+```yaml
+search_backend: foo.registry.index.xapian
+```
+
+In this case, the module is imported, and an instance of it's `Index`
+class is used as the search backend.
+
 ### Email options
 
 Settings these options makes the Registry send an email on each code Exception:
@@ -408,3 +441,9 @@ pip install tox
 cd docker-registry/
 tox
 ```
+
+[search-endpoint]:
+  http://docs.docker.io/en/latest/reference/api/index_api/#get--v1-search
+[SQLAlchemy]: http://docs.sqlalchemy.org/
+[create_engine]:
+  http://docs.sqlalchemy.org/en/latest/core/engines.html#sqlalchemy.create_engine

config/config_sample.yml

@@ -31,6 +31,10 @@ common:
     # Let gunicorn set this environment variable or set a random string here
     secret_key: _env:SECRET_KEY
 
+    search_backend: "_env:SEARCH_BACKEND:"
+    sqlalchemy_index_database:
+        "_env:SQLALCHEMY_INDEX_DATABASE:sqlite:////tmp/docker-registry.db"
+
 
 # This is the default configuration when no flavor is specified
 dev:

config/config_test.yml

@@ -15,3 +15,6 @@ test:
     gs_access_key: _env:GS_ACCESS_KEY
     gs_secret_key: _env:GS_SECRET_KEY 
     gs_secure: false
+
+    search_backend: sqlalchemy
+    sqlalchemy_index_database: sqlite:////tmp/docker-registry.db

lib/index/__init__.py

@@ -0,0 +1,86 @@
+"""Index backends for the search endpoint
+"""
+
+import importlib
+import signals
+
+import config
+
+
+__all__ = ['load']
+
+
+class Index (object):
+    """A backend for the search endpoint
+
+    The backend can use .walk_storage to generate an initial index,
+    ._handle_repository_* to stay up to date with registry changes,
+    and .results to respond to queries.
+    """
+    def __init__(self):
+        signals.repository_created.connect(self._handle_repository_created)
+        signals.repository_updated.connect(self._handle_repository_updated)
+        signals.repository_deleted.connect(self._handle_repository_deleted)
+
+    def _walk_storage(self, store):
+        """Iterate through repositories in storage
+
+        This helper is useful for building an initial database for
+        your search index.  Yields dictionaries:
+
+          {'name': name, 'description': description}
+        """
+        try:
+            namespace_paths = list(
+                store.list_directory(path=store.repositories))
+        except OSError:
+            namespace_paths = []
+        for namespace_path in namespace_paths:
+            namespace = namespace_path.rsplit('/', 1)[-1]
+            try:
+                repository_paths = list(
+                    store.list_directory(path=namespace_path))
+            except OSError:
+                repository_paths = []
+            for path in repository_paths:
+                repository = path.rsplit('/', 1)[-1]
+                name = '{0}/{1}'.format(namespace, repository)
+                description = None  # TODO(wking): store descriptions
+                yield({'name': name, 'description': description})
+
+    def _handle_repository_created(
+            self, sender, namespace, repository, value):
+        pass
+
+    def _handle_repository_updated(
+            self, sender, namespace, repository, value):
+        pass
+
+    def _handle_repository_deleted(self, sender, namespace, repository):
+        pass
+
+    def results(self, search_term):
+        """Return a list of results matching search_term
+
+        The list elements should be dictionaries:
+
+          {'name': name, 'description': description}
+        """
+        raise NotImplementedError('results method for {!r}'.format(self))
+
+
+def load(kind=None):
+    """Returns an Index instance according to the configuration."""
+    cfg = config.load()
+    if not kind:
+        kind = cfg.search_backend.lower()
+    if kind == 'sqlalchemy':
+        from . import db
+        return db.SQLAlchemyIndex()
+    try:
+        module = importlib.import_module(kind)
+    except ImportError:
+        pass
+    else:
+        return module.Index()
+    raise NotImplementedError('Unknown index type {0!r}'.format(kind))

lib/index/db.py

@@ -0,0 +1,125 @@
+"""An SQLAlchemy backend for the search endpoint
+"""
+
+import sqlalchemy
+import sqlalchemy.exc
+import sqlalchemy.ext.declarative
+import sqlalchemy.orm
+import sqlalchemy.sql.functions
+
+import config
+import storage
+
+from . import Index
+
+
+Base = sqlalchemy.ext.declarative.declarative_base()
+
+
+class Version (Base):
+    "Schema version for the search-index database"
+    __tablename__ = 'version'
+
+    id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
+
+    def __repr__(self):
+        return '<{0}(id={1})>'.format(type(self).__name__, self.id)
+
+
+class Repository (Base):
+    "Repository description"
+    __tablename__ = 'repository'
+
+    id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
+    name = sqlalchemy.Column(
+        sqlalchemy.String, nullable=False, unique=True)
+    description = sqlalchemy.Column(sqlalchemy.String)
+
+    def __repr__(self):
+        return "<{0}(name='{1}', description='{2}')>".format(
+            type(self).__name__, self.name, self.description)
+
+
+class SQLAlchemyIndex (Index):
+    """Maintain an index of repository data
+
+    The index is a dictionary.  The keys are
+    '{namespace}/{repository}' strings, and the values are description
+    strings.  For example:
+
+      index['library/ubuntu'] = 'An ubuntu image...'
+    """
+    def __init__(self, database=None):
+        if database is None:
+            cfg = config.load()
+            database = cfg.sqlalchemy_index_database
+        self._engine = sqlalchemy.create_engine(database)
+        self._session = sqlalchemy.orm.sessionmaker(bind=self._engine)
+        self.version = 1
+        self._setup_database()
+        super(SQLAlchemyIndex, self).__init__()
+
+    def _setup_database(self):
+        session = self._session()
+        try:
+            version = session.query(
+                sqlalchemy.sql.functions.max(Version.id)).first()[0]
+        except sqlalchemy.exc.OperationalError:
+            version = None
+        if version:
+            if version != self.version:
+                raise NotImplementedError(
+                    'unrecognized search index version {0}'.format(version))
+        else:
+            self._generate_index(session=session)
+        session.close()
+
+    def _generate_index(self, session):
+        store = storage.load()
+        Base.metadata.create_all(self._engine)
+        session.add(Version(id=self.version))
+        for repository in self._walk_storage(store=store):
+            session.add(Repository(**repository))
+        session.commit()
+
+    def _handle_repository_created(
+            self, sender, namespace, repository, value):
+        name = '{0}/{1}'.format(namespace, repository)
+        description = ''  # TODO(wking): store descriptions
+        session = self._session()
+        session.add(Repository(name=name, description=description))
+        session.commit()
+        session.close()
+
+    def _handle_repository_updated(
+            self, sender, namespace, repository, value):
+        name = '{0}/{1}'.format(namespace, repository)
+        description = ''  # TODO(wking): store descriptions
+        session = self._session()
+        session.query(Repository).filter(
+            Repository.name == name).update(
+                values={'description': description},
+                synchronize_session=False)
+        session.commit()
+        session.close()
+
+    def _handle_repository_deleted(self, sender, namespace, repository):
+        name = '{0}/{1}'.format(namespace, repository)
+        session = self._session()
+        session.query(Repository).filter(Repository.name == name).delete()
+        session.commit()
+        session.close()
+
+    def results(self, search_term):
+        session = self._session()
+        like_term = '%{}%'.format(search_term)
+        repositories = session.query(Repository).filter(
+            sqlalchemy.sql.or_(
+                Repository.name.like(like_term),
+                Repository.description.like(like_term)))
+        return [
+            {
+                'name': repo.name,
+                'description': repo.description,
+            }
+            for repo in repositories]

registry/__init__.py

@@ -4,6 +4,7 @@
 from .app import app
 from .tags import *
 from .images import *
+from . import search as search
 from .status import *
 
 import config

registry/index.py

@@ -133,8 +133,3 @@ def delete_repository_images(namespace, repository):
 @toolkit.parse_repository_name
 def put_repository_auth(namespace, repository):
     return toolkit.response('OK')
-
-
-@app.route('/v1/search', methods=['GET'])
-def get_search():
-    return toolkit.response({})

registry/search.py

@@ -0,0 +1,30 @@
+import flask
+
+import config
+import lib.index as index
+
+from . import toolkit
+from .app import app
+
+
+cfg = config.load()
+
+# Enable the search index
+if cfg.search_backend:
+    INDEX = index.load(cfg.search_backend.lower())
+else:
+    INDEX = None
+
+
+@app.route('/v1/search', methods=['GET'])
+def get_search():
+    search_term = flask.request.args.get('q', '')
+    if INDEX is None:
+        results = []
+    else:
+        results = INDEX.results(search_term=search_term)
+    return toolkit.response({
+        'query': search_term,
+        'num_results': len(results),
+        'results': results,
+    })

requirements.txt

@@ -12,5 +12,6 @@ python-keystoneclient==0.3.1
 python-swiftclient==1.8.0
 redis==2.8.0
 rsa==3.1.2
+sqlalchemy==0.9.2
 bugsnag==1.3.1
 gcs-oauth2-boto-plugin>=1.3

run.sh

@@ -6,4 +6,4 @@ GUNICORN_GRACEFUL_TIMEOUT=${GUNICORN_GRACEFUL_TIMEOUT:-3600}
 GUNICORN_SILENT_TIMEOUT=${GUNICORN_SILENT_TIMEOUT:-3600}
 
 cd "$(dirname $0)"
-exec gunicorn --access-logfile - --debug --max-requests 100 --graceful-timeout $GUNICORN_GRACEFUL_TIMEOUT -t $GUNICORN_SILENT_TIMEOUT -k gevent -b 0.0.0.0:$REGISTRY_PORT -w $GUNICORN_WORKERS wsgi:application
+exec gunicorn --access-logfile - --max-requests 100 --graceful-timeout $GUNICORN_GRACEFUL_TIMEOUT -t $GUNICORN_SILENT_TIMEOUT -k gevent -b 0.0.0.0:$REGISTRY_PORT -w $GUNICORN_WORKERS $GUNICORN_OPTS "$@" wsgi:application