index: Factor out the SQLAlchemy Index into flexible backends

Make it easy to add alternative backends for the search endpoint.
I've added a 'search_backend' config and renamed the 'search_index' to
'search_sqlalchemy_database' to avoid colliding with other backends.

I've called the SQLAlchemy module 'db' so we can distinguish between
the global sqlalchemy package and our SQLAlchemy-based index module
without using the absolute_import future [1,2,3,4].

[1]: https://docs.python.org/2/library/__future__.html
[2]: http://legacy.python.org/dev/peps/pep-0328/
[3]: #247 (comment)
[4]: #247 (comment)

Reported-by: Lee Trout <[email protected]>

Author: wking

README.md

@@ -46,7 +46,9 @@ in action in the example below...
 
 common:
     loglevel: info
-    search_index: "_env:SEARCH_INDEX:"
+    search_backend: "_env:SEARCH_BACKEND:"
+    sqlalchemy_index_database:
+        "_env:SQLALCHEMY_INDEX_DATABASE:sqlite:////tmp/docker-registry.db"
 
 prod:
     loglevel: warn
@@ -207,19 +209,23 @@ dev:
 
 ### Search-engine options
 
-The Docker Registry can optionally use [SQLAlchemy][] to index
-repository information in a database, for the `GET /v1/search`
-[endpoint][search-endpoint].  You can configure the backend with a
-configuration like:
+The Docker Registry can optionally index repository information in a
+database for the `GET /v1/search` [endpoint][search-endpoint].  You
+can configure the backend with a configuration like:
 
 ```yaml
-search_index: "_env:SEARCH_INDEX:"
+search_backend: "_env:SEARCH_BACKEND:"
 ```
 
-you can use the `SEARCH_INDEX` environment variable to configure the
-string passed to [create_engine][].  If the configured `search_index`
-is empty, no index is built, and the search endpoint always returns
-empty results.
+The `search_backend` setting selects the search backend to use.  If
+`search_backend` is empty, no index is built, and the search endpoint
+always returns empty results.  Currently supported backends and their
+backend-specific configuration options are:
+
+* `sqlalchemy': Use [SQLAlchemy][].
+    * The backing database is selected with
+      `sqlalchemy_index_database`, which is passed through to
+      [create_engine][].
 
 ### Email options
 
@@ -422,8 +428,8 @@ cd docker-registry/
 tox
 ```
 
-[SQLAlchemy]: http://docs.sqlalchemy.org/
 [search-endpoint]:
   http://docs.docker.io/en/latest/reference/api/index_api/#get--v1-search
+[SQLAlchemy]: http://docs.sqlalchemy.org/
 [create_engine]:
   http://docs.sqlalchemy.org/en/latest/core/engines.html#sqlalchemy.create_engine

config/config_sample.yml

@@ -31,7 +31,9 @@ common:
     # Let gunicorn set this environment variable or set a random string here
     secret_key: _env:SECRET_KEY
 
-    search_index: "_env:SEARCH_INDEX:"
+    search_backend: "_env:SEARCH_BACKEND:"
+    sqlalchemy_index_database:
+        "_env:SQLALCHEMY_INDEX_DATABASE:sqlite:////tmp/docker-registry.db"
 
 
 # This is the default configuration when no flavor is specified

config/config_test.yml

@@ -15,4 +15,5 @@ test:
     gs_secret_key: _env:GS_SECRET_KEY 
     gs_secure: false
 
-    search_index: sqlite:////tmp/docker-registry.db
+    search_backend: sqlalchemy
+    sqlalchemy_index_database: sqlite:////tmp/docker-registry.db

lib/index/__init__.py

@@ -0,0 +1,80 @@
+"""Index backends for the search endpoint
+"""
+
+import signals
+
+import config
+
+
+__all__ = ['load']
+
+
+class Index (object):
+    """A backend for the search endpoint
+
+    The backend can use .walk_storage to generate an initial index,
+    ._handle_repository_* to stay up to date with registry changes,
+    and .results to respond to queries.
+    """
+    def __init__(self):
+        signals.repository_created.connect(self._handle_repository_created)
+        signals.repository_updated.connect(self._handle_repository_updated)
+        signals.repository_deleted.connect(self._handle_repository_deleted)
+
+    def _walk_storage(self, store):
+        """Iterate through repositories in storage
+
+        This helper is useful for building an initial database for
+        your search index.  Yields dictionaries:
+
+          {'name': name, 'description': description}
+        """
+        try:
+            namespace_paths = list(
+                store.list_directory(path=store.repositories))
+        except OSError:
+            namespace_paths = []
+        for namespace_path in namespace_paths:
+            namespace = namespace_path.rsplit('/', 1)[-1]
+            try:
+                repository_paths = list(
+                    store.list_directory(path=namespace_path))
+            except OSError:
+                repository_paths = []
+            for path in repository_paths:
+                repository = path.rsplit('/', 1)[-1]
+                name = '{0}/{1}'.format(namespace, repository)
+                description = None  # TODO(wking): store descriptions
+                yield({'name': name, 'description': description})
+
+    def _handle_repository_created(
+            self, sender, namespace, repository, value):
+        pass
+
+    def _handle_repository_updated(
+            self, sender, namespace, repository, value):
+        pass
+
+    def _handle_repository_deleted(self, sender, namespace, repository):
+        pass
+
+    def results(self, search_term):
+        """Return a list of results matching search_term
+
+        The list elements should be dictionaries:
+
+          {'name': name, 'description': description}
+        """
+        raise NotImplementedError('results method for {!r}'.format(self))
+
+
+def load(kind=None):
+    """Returns an Index instance according to the configuration."""
+    cfg = config.load()
+    if not kind:
+        kind = cfg.search_backend.lower()
+    if kind == 'sqlalchemy':
+        from . import db
+        return db.SQLAlchemyIndex()
+    else:
+        raise NotImplementedError('Unknown index type {0!r}'.format(kind))

lib/index/db.py

@@ -0,0 +1,125 @@
+"""An SQLAlchemy backend for the search endpoint
+"""
+
+import sqlalchemy
+import sqlalchemy.exc
+import sqlalchemy.ext.declarative
+import sqlalchemy.orm
+import sqlalchemy.sql.functions
+
+import config
+import storage
+
+from . import Index
+
+
+Base = sqlalchemy.ext.declarative.declarative_base()
+
+
+class Version (Base):
+    "Schema version for the search-index database"
+    __tablename__ = 'version'
+
+    id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
+
+    def __repr__(self):
+        return '<{0}(id={1})>'.format(type(self).__name__, self.id)
+
+
+class Repository (Base):
+    "Repository description"
+    __tablename__ = 'repository'
+
+    id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True)
+    name = sqlalchemy.Column(
+        sqlalchemy.String, nullable=False, unique=True)
+    description = sqlalchemy.Column(sqlalchemy.String)
+
+    def __repr__(self):
+        return "<{0}(name='{1}', description='{2}')>".format(
+            type(self).__name__, self.name, self.description)
+
+
+class SQLAlchemyIndex (Index):
+    """Maintain an index of repository data
+
+    The index is a dictionary.  The keys are
+    '{namespace}/{repository}' strings, and the values are description
+    strings.  For example:
+
+      index['library/ubuntu'] = 'An ubuntu image...'
+    """
+    def __init__(self, database=None):
+        if database is None:
+            cfg = config.load()
+            database = cfg.sqlalchemy_index_database
+        self._engine = sqlalchemy.create_engine(database)
+        self._session = sqlalchemy.orm.sessionmaker(bind=self._engine)
+        self.version = 1
+        self._setup_database()
+        super(SQLAlchemyIndex, self).__init__()
+
+    def _setup_database(self):
+        session = self._session()
+        try:
+            version = session.query(
+                sqlalchemy.sql.functions.max(Version.id)).first()[0]
+        except sqlalchemy.exc.OperationalError:
+            version = None
+        if version:
+            if version != self.version:
+                raise NotImplementedError(
+                    'unrecognized search index version {0}'.format(version))
+        else:
+            self._generate_index(session=session)
+        session.close()
+
+    def _generate_index(self, session):
+        store = storage.load()
+        Base.metadata.create_all(self._engine)
+        session.add(Version(id=self.version))
+        for repository in self._walk_storage(store=store):
+            session.add(Repository(**repository))
+        session.commit()
+
+    def _handle_repository_created(
+            self, sender, namespace, repository, value):
+        name = '{0}/{1}'.format(namespace, repository)
+        description = ''  # TODO(wking): store descriptions
+        session = self._session()
+        session.add(Repository(name=name, description=description))
+        session.commit()
+        session.close()
+
+    def _handle_repository_updated(
+            self, sender, namespace, repository, value):
+        name = '{0}/{1}'.format(namespace, repository)
+        description = ''  # TODO(wking): store descriptions
+        session = self._session()
+        session.query(Repository).filter(
+            Repository.name == name).update(
+                values={'description': description},
+                synchronize_session=False)
+        session.commit()
+        session.close()
+
+    def _handle_repository_deleted(self, sender, namespace, repository):
+        name = '{0}/{1}'.format(namespace, repository)
+        session = self._session()
+        session.query(Repository).filter(Repository.name == name).delete()
+        session.commit()
+        session.close()
+
+    def results(self, search_term):
+        session = self._session()
+        like_term = '%{}%'.format(search_term)
+        repositories = session.query(Repository).filter(
+            sqlalchemy.sql.or_(
+                Repository.name.like(like_term),
+                Repository.description.like(like_term)))
+        return [
+            {
+                'name': repo.name,
+                'description': repo.description,
+            }
+            for repo in repositories]