[refactor] Made managers pluggable

Author: buske

mme_server/auth.py

@@ -6,162 +6,35 @@
 """
 from __future__ import with_statement, division, unicode_literals
 
-import json
 import logging
 import flask
 
 from functools import wraps
 
 from flask import request, jsonify
-from elasticsearch import Elasticsearch
-from elasticsearch_dsl import Search, Q
 
-from .datastore import ESIndex
-from .models import get_backend
+from .backend import get_backend
 
 
 logger = logging.getLogger(__name__)
 
 
-def get_server_manager():
-    manager = getattr(flask.g, '_mme_servers', None)
-    if manager is None:
-        manager = flask.g._mme_servers = ServerManager()
-
-    return manager
-
-
 def auth_token_required():
     def decorator(f):
         @wraps(f)
         def decorated_function(*args, **kwargs):
             logger.info("Authenticating request")
             token = request.headers.get('X-Auth-Token')
-            auth = get_server_manager()
-            server = auth.verify(token)
+            backend = get_backend()
+            servers = backend.get_manager('servers')
+            server = servers.verify(token)
             if not server:
                 error = jsonify(message='X-Auth-Token not authorized')
                 error.status_code = 401
                 return error
 
+            # Set authenticated server as flask global for request
             flask.g.server = server
             return f(*args, **kwargs)
         return decorated_function
     return decorator
-
-
-class ServerManager:
-    FIELDS = ['server_id', 'server_label', 'server_key', 'direction', 'base_url']
-    DOC_CONFIG = {
-        'properties': {
-            'server_id': {
-                'type': 'string',
-                'index': 'not_analyzed',
-            },
-            'server_label': {
-                'type': 'string',
-                'index': 'not_analyzed',
-            },
-            'server_key': {
-                'type': 'string',
-                'index': 'not_analyzed',
-            },
-            'direction': {
-                'type': 'string',
-                'index': 'not_analyzed',
-            },
-            'base_url': {
-                'type': 'string',
-                'index': 'not_analyzed',
-            }
-        }
-    }
-
-    def __init__(self, backend=None):
-        if backend is None:
-            backend = Elasticsearch()
-
-        self._index = ESIndex(db=backend,
-                              name='servers',
-                              doc_type='server',
-                              doc_config=self.DOC_CONFIG)
-
-    @property
-    def index(self):
-        return self._index
-
-    def add(self, server_id, server_label, server_key, direction, base_url):
-        assert server_id and server_key and direction in ['in', 'out']
-
-        if base_url and not base_url.startswith('https://'):
-            logger.error('base URL must start with "https://"')
-            return
-
-        self.index.ensure_exists()
-
-        data = {
-            'server_id': server_id,
-            'server_label': server_label,
-            'server_key': server_key,
-            'direction': direction,
-            'base_url': base_url,
-        }
-
-        # If it already exists, update
-        s = self.index.search()
-        s = s.filter('term', server_id=server_id)
-        s = s.filter('term', direction=direction)
-        results = s.execute()
-
-        if results.hits.total > 1:
-            logger.error('Found two or more matching server entries')
-        else:
-            id = None
-            if results.hits.total == 1:
-                # Found a match, so update instead
-                id = results.hits[0].meta.id
-
-            self.index.save(id=id, doc=data)
-            logger.info("Authorized server:\n{}".format(json.dumps(data, indent=4, sort_keys=True)))
-            # Refresh index to ensure immediately usable
-            self.index.refresh()
-
-    def remove(self, server_id, direction):
-        if self.index.exists():
-            s = self.index.search()
-            s = s.filter('term', server_id=server_id)
-
-            if direction:
-                s = s.filter('term', direction=direction)
-            results = s.execute()
-
-            for hit in results:
-                id = hit.meta.id
-                self.index.delete(id=id)
-                logger.info("Deleted server:{} direction:{}".format(hit.server_id, hit.direction))
-
-    def list(self):
-        rows = []
-        if self.index.exists():
-            s = self.index.search()
-            s = s.query('match_all')
-
-            # Iterate through all, using scan
-            for hit in s.scan():
-                row = dict([(field, hit[field]) for field in self.FIELDS])
-                rows.append(row)
-
-        return {
-            'fields': self.FIELDS,
-            'rows': rows
-        }
-
-    def verify(self, key):
-        if key and self.index.exists():
-            s = self.index.search()
-            s = s.filter('term', server_key=key)
-            s = s.filter('term', direction='in')
-            results = s.execute()
-
-            if results.hits:
-                return results.hits[0]

mme_server/backend.py

@@ -0,0 +1,23 @@
+"""
+Module for accessing the backend connection
+"""
+
+from __future__ import with_statement, division, unicode_literals
+
+import logging
+import flask
+
+from elasticsearch import Elasticsearch
+
+from .managers import Managers
+
+logger = logging.getLogger(__name__)
+
+
+
+def get_backend():
+    backend = getattr(flask.g, '_mme_backend', None)
+    if backend is None:
+        backend = flask.g._mme_backend = Managers(Elasticsearch())
+
+    return backend

mme_server/cli.py

@@ -10,9 +10,8 @@
 
 from binascii import hexlify
 
-from .auth import get_server_manager
+from .backend import get_backend
 from .compat import urlretrieve
-from .models import get_backend
 from .server import app
 
 
@@ -59,10 +58,12 @@ def index_file(index, filename, url):
 
     with app.app_context():
         backend = get_backend()
+        patients = backend.get_manager('patients')
+        vocabularies = backend.get_manager('vocabularies')
         index_funcs = {
-            'hpo': backend.vocabularies.index_hpo,
-            'genes': backend.vocabularies.index_genes,
-            'patients': backend.patients.index_file,
+            'hpo': vocabularies.index_hpo,
+            'genes': vocabularies.index_genes,
+            'patients': patients.index_file,
         }
         index_funcs[index](filename=filename)
 
@@ -76,64 +77,83 @@ def fetch_resource(filename, url):
         logger.info('Saved file to: {}'.format(filename))
 
 
-def list_servers():
+def list_servers(direction='out'):
     with app.app_context():
-        servers = get_server_manager()
-        response = servers.list()
+        backend = get_backend()
+        servers = backend.get_manager('servers')
+        response = servers.list(direction=direction)
         # print header
         fields = response['fields']
         print('\t'.join(fields))
 
         for server in response.get('rows', []):
             print('\t'.join([repr(server[field]) for field in fields]))
 
+def list_clients():
+    return list_servers(direction='in')
 
-def add_server(id, direction, key=None, label=None, base_url=None):
+def add_server(id, direction='out', key=None, label=None, base_url=None):
     if not label:
         label = id
 
     if direction == 'out' and not base_url:
         raise Exception('--base-url must be specified for outgoing servers')
 
     with app.app_context():
-        servers = get_server_manager()
+        backend = get_backend()
+        servers = backend.get_manager('servers')
         # Generate a random key if one was not provided
         if key is None:
             key = hexlify(os.urandom(30)).decode()
         servers.add(server_id=id, server_key=key, direction=direction, server_label=label, base_url=base_url)
 
+def add_client(id, key=None, label=None):
+    add_server(id, 'in', key=key, label=label)
 
-def remove_server(id, direction):
+def remove_server(id, direction='out'):
     with app.app_context():
-        servers = get_server_manager()
+        backend = get_backend()
+        servers = backend.get_manager('servers')
         servers.remove(server_id=id, direction=direction)
 
+def remove_client(id):
+    remove_server(id, direction='in')
+
 
 def run_tests():
     suite = unittest.TestLoader().discover('.'.join([__package__, 'tests']))
     unittest.TextTestRunner().run(suite)
 
 
-def add_auth_parser(parser):
-    auth_parser = parser.add_parser('auth', description="Authorization sub-commands")
-    subparsers = auth_parser.add_subparsers(title='subcommands')
-    subparser = subparsers.add_parser('add', description="Add server authorization")
-    subparser.add_argument("id", help="A unique server identifier")
-    subparser.add_argument("direction", choices=["in", "out"],
-        help="Direction of server authorization, 'in': the other server can send requests, 'out': this server can send requests")
-    subparser.add_argument("--key", help="The secret key used to authenticate requests to/from the server (default: randomly generate a secure key)")
-    subparser.add_argument("--label", help="The display name for the server ")
-    subparser.add_argument("--base-url", dest="base_url", help="The base URL for sending API requests to the other server (e.g., <base-url>/match should be a valid endpoint). Must be specified for outgoing requests")
-    subparser.set_defaults(function=add_server)
+def add_server_subcommands(parser, direction):
+    """Add subparser for incoming or outgoing servers
+
+    direction - 'in': incoming servers, 'out': outgoing servers
+    """
+    server_type = 'client' if direction == 'in' else 'server'
+    subparsers = parser.add_subparsers(title='subcommands')
+    subparser = subparsers.add_parser('add', description="Add {} authorization".format(server_type))
+    subparser.add_argument("id", help="A unique {} identifier".format(server_type))
+    subparser.add_argument("--key", help="The secret key used to authenticate requests to/from the {} (default: randomly generate a secure key)".format(server_type))
+    subparser.add_argument("--label", help="The display name for the {}".format(server_type))
+    if server_type == 'server':
+        subparser.add_argument("--base-url", dest="base_url", help="The base URL for sending API requests to the server (e.g., <base-url>/match should be a valid endpoint). Must be specified for outgoing requests")
+        subparser.set_defaults(function=add_server)
+    else:
+        subparser.set_defaults(function=add_client)
 
-    subparser = subparsers.add_parser('rm', description="Remove server authorization")
-    subparser.add_argument("id", help="Server identifier")
-    subparser.add_argument("direction", choices=["in", "out"],
-        help="Direction of server authorization, 'in': the other server can send requests, 'out': this server can send requests")
-    subparser.set_defaults(function=remove_server)
+    subparser = subparsers.add_parser('rm', description="Remove {} authorization".format(server_type))
+    subparser.add_argument("id", help="The {} identifier".format(server_type))
+    if server_type == 'server':
+        subparser.set_defaults(function=remove_server)
+    else:
+        subparser.set_defaults(function=remove_client)
 
-    subparser = subparsers.add_parser('list', description="List all existing server authorizations")
-    subparser.set_defaults(function=list_servers)
+    subparser = subparsers.add_parser('list', description="List {} authorizations".format(server_type))
+    if server_type == 'server':
+        subparser.set_defaults(function=list_servers)
+    else:
+        subparser.set_defaults(function=list_clients)
 
 
 def parse_args(args):
@@ -174,7 +194,11 @@ def parse_args(args):
                            help="The host the server will listen to (0.0.0.0 to listen globally; 127.0.0.1 to listen locally; default: %(default)s)")
     subparser.set_defaults(function=app.run)
 
-    add_auth_parser(subparsers)
+    subparser = subparsers.add_parser('servers', description="Server authorization sub-commands")
+    add_server_subcommands(subparser, direction='out')
+
+    subparser = subparsers.add_parser('clients', description="Client authorization sub-commands")
+    add_server_subcommands(subparser, direction='in')
 
     subparser = subparsers.add_parser('test', description="Run tests")
     subparser.set_defaults(function=run_tests)