Merge branch 'develop'

Author: DavidLemayian

README.md

@@ -22,25 +22,45 @@ Clone the repo from github `$ git clone [email protected]:RyanSept/HealthTools.KE-a
 
 Change directory into package `$ cd HealthTools.KE-api`
 
-Install the dependencies by running `$ pip install requirements.txt`
+Install the dependencies by running `$ pip install -r requirements.txt`
+
+Install Memcached
+ * If on linux follow this [link](https://github.com/memcached/memcached/wiki/Install)
+ * On mac use `brew install memcached`
 
 You can set the required environment variables like so
 ```<>
+$ export APP_DEBUG=<False> # True or False
+$ export MEMCACHED_URL=<memcache_url:port> # defaults to 127.0.0.1:8000
 $ export GA_TRACKING_ID=<google-analytics-tracking-id>
 $ export SMS_USER=<sms-provider-user-id>
 $ export SMS_PASS=<sms-provider-passcode>
 $ export SMS_SHORTCODE=<sms-provider-shortcode>
 $ export SMS_SEND_URL=<url-for-sms-provider>
-$ export CONFIG=<config-mode>  # eg. "healthtools_ke_api.settings.DevConfig"
-$ export AWS_ACCESS_KEY_ID=<aws-access-key-id>
+$ export AWS_ACCESS_KEY=<aws-access-key-id>
 $ export AWS_SECRET_KEY=<aws-secret-key>
-$ export AWS_REGION="<aws-region>
+$ export ES_HOST=<elasticsearch_host_endpoint> (DO NOT SET THIS IF YOU WOULD LIKE TO USE ELASTIC SEARCH LOCALLY ON YOUR MACHINE)
+$export ES_PORT=<elasticsearch_port>
 ```
+**If you want to use elasticsearch locally on your machine use the following instructions to set it up**
+
+For linux and windows users, follow instructions from this [link](https://www.elastic.co/guide/en/elasticsearch/reference/current/setup.html)
+
+For mac users run `brew install elasticsearch` on your terminal
+
+Run memcached on your terminal `$ memcached -p <port you set MEMCACHED_URL to run on>(default: 8000)`
+
+If you set up elasticsearch locally run it `$ elasticsearch`
 
 You can now run the server `$ python manage.py` or `gunicorn manage:app` for production.
 
 
+
 ## Running the tests
 
+Run memcached on your terminal `$ memcached -p <port you set MEMCACHED_URL to run on>(default: 8000)`
+
+_**make sure if you use elasticsearch locally, it's running**_
+
 Use nosetests to run tests (with stdout) like this:
-```$ nosetests --nocapture```
+```$ nosetests --nocapture```

healthtools_ke_api/__init__.py

@@ -12,11 +12,6 @@
 
 
 app = Flask(__name__)
-try:
-    app.config.from_object(os.getenv('CONFIG'))
-except KeyError:
-    print "No config has been specified for use in the environment variables."
-    sys.exit()
 
 app.register_blueprint(doctors_api, url_prefix='/doctors')
 app.register_blueprint(nurses_api, url_prefix='/nurses')

healthtools_ke_api/settings.py

@@ -1,33 +1,28 @@
 import os
 
+DEBUG = os.getenv("APP_DEBUG", False)
+
 # Url of memcached server
-MEMCACHED_URL = os.getenv("MEMCACHED_URL")
+MEMCACHED_URL = os.getenv("MEMCACHED_URL", "127.0.0.1:11211")
 
 # Amazon Web Services configs
-AWS_CONFIGS = {
-    "aws_access_key_id": os.getenv("AWS_ACCESS_KEY_ID"),
-    "aws_secret_access_key": os.getenv("AWS_SECRET_KEY"),
-    "region_name": os.getenv("AWS_REGION"),
+AWS = {
+    "access_key": os.getenv("AWS_ACCESS_KEY"),
+    "secret_key": os.getenv("AWS_SECRET_KEY"),
+    "region": os.getenv("AWS_REGION", "eu-west-1"),
 }
 
+# Elastic Search configs
+ES = {
+    "host": os.getenv("ES_HOST", "127.0.0.1"),
+    "port": os.getenv("ES_PORT", "9200"),
+    "index": "healthtools-ke"
+}
 
-class Config(object):
-    # Google Analytics tracking id
-    GA_TRACKING_ID = os.environ.get('GA_TRACKING_ID')
-
-    # SMS provider credentials
-    SMS_USER = os.environ.get('SMS_USER')
-    SMS_PASS = os.environ.get('SMS_PASS')
-    SMS_SHORTCODE = os.environ.get('SMS_SHORTCODE')
-
-    DOCTORS_SEARCH_URL = "https://6ujyvhcwe6.execute-api.eu-west-1.amazonaws.com/prod"
-    NURSE_SEARCH_URL = "https://api.healthtools.codeforafrica.org/nurses/search.json"
-    CO_SEARCH_URL = "https://vfblk3b8eh.execute-api.eu-west-1.amazonaws.com/prod"
-    NHIF_SEARCH_URL = "https://t875kgqahj.execute-api.eu-west-1.amazonaws.com/prod"
-    HF_SEARCH_URL = "https://187mzjvmpd.execute-api.eu-west-1.amazonaws.com/prod"
-
+# Google Analytics tracking id
+GA_TRACKING_ID = os.environ.get('GA_TRACKING_ID')
 
-# development config
-class DevConfig(Config):
-    DEBUG = True
-    SERVER_NAME = "127.0.0.1:8000"
+# SMS provider credentials
+SMS_USER = os.environ.get('SMS_USER')
+SMS_PASS = os.environ.get('SMS_PASS')
+SMS_SHORTCODE = os.environ.get('SMS_SHORTCODE')

healthtools_ke_api/tests/test_clinical_officers_api.py

@@ -1,14 +1,15 @@
 from unittest import TestCase
 from healthtools_ke_api import app
-from healthtools_ke_api.views.clinical_officers import get_clinical_officers_from_cloudsearch
+from healthtools_ke_api.views.elastic_search import Elastic
 
 
 class TestClinicalOfficersAPI(TestCase):
     def setUp(self):
         self.client = app.test_client()
+        self.es = Elastic()
 
-    def test_gets_cos_from_cloudsearch(self):
-        clinical_officers = get_clinical_officers_from_cloudsearch("Marie")
+    def test_gets_cos_from_elasticsearch(self):
+        clinical_officers = self.es.get_from_elasticsearch("clinical-officers", "Jacob")
         self.assertTrue(len(clinical_officers) > 0)
 
     def test_cos_endpoint_with_bad_query(self):

healthtools_ke_api/tests/test_doctors_api.py

@@ -1,14 +1,15 @@
 from unittest import TestCase
 from healthtools_ke_api import app
-from healthtools_ke_api.views.doctors import get_doctors_from_cloudsearch
+from healthtools_ke_api.views.elastic_search import Elastic
 
 
 class TestDoctorsAPI(TestCase):
     def setUp(self):
         self.client = app.test_client()
+        self.es = Elastic()
 
-    def test_gets_doctors_from_cloudsearch(self):
-        doctors = get_doctors_from_cloudsearch("Marie")
+    def test_gets_doctors_from_elasticsearch(self):
+        doctors = self.es.get_from_elasticsearch("doctors", "BHATT")
         self.assertTrue(len(doctors) > 0)
 
     def test_doctors_endpoint_with_bad_query(self):

healthtools_ke_api/views/clinical_officers.py

@@ -1,15 +1,9 @@
-import boto3
 from flask import Blueprint, request, jsonify, current_app
 
+from elastic_search import Elastic
 from healthtools_ke_api.analytics import track_event
-from healthtools_ke_api.settings import AWS_CONFIGS
-
 
 clinical_officers_api = Blueprint('clinical_officers_api', __name__)
-COS_CLOUDSEARCH_ENDPOINT = "http://doc-cfa-healthtools-ke-cos-nhxtw3w5goufkzram4er7sciz4.eu-west-1.cloudsearch.amazonaws.com/"
-cloudsearch_client = boto3.client("cloudsearchdomain",
-                                  endpoint_url=COS_CLOUDSEARCH_ENDPOINT,
-                                  **AWS_CONFIGS)
 
 
 @clinical_officers_api.route('/', methods=['GET'])
@@ -21,7 +15,9 @@ def index():
         "name": "API to Kenyan Clinical Officers registry",
         "authentication": [],
         "endpoints": {
-            "/": {"methods": ["GET"]},
+            "/": {
+                "methods": ["GET"]
+            },
             "/clinical-officers/search.json": {
                 "methods": ["GET"],
                 "args": {
@@ -46,7 +42,8 @@ def search():
 
         # get clinical_officers by that name from aws
         response = {}
-        clinical_officers = get_clinical_officers_from_cloudsearch(query)
+        es = Elastic()
+        clinical_officers = es.get_from_elasticsearch('clinical-officers', query)
 
         if not clinical_officers:
             response["message"] = "No clinical officer by that name found."
@@ -65,11 +62,3 @@ def search():
             "message": str(err),
             "data": {"clinical_officers": []}
         })
-
-
-def get_clinical_officers_from_cloudsearch(query):
-    '''
-    Get clinical officers from AWS cloudsearch
-    '''
-    results = cloudsearch_client.search(query=query, size=10000)
-    return results["hits"]["hit"]

healthtools_ke_api/views/doctors.py

@@ -1,15 +1,9 @@
-import boto3
 from flask import Blueprint, request, jsonify, current_app
 
 from healthtools_ke_api.analytics import track_event
-from healthtools_ke_api.settings import AWS_CONFIGS
-
+from elastic_search import Elastic
 
 doctors_api = Blueprint('doctors_api', __name__)
-DOCTORS_CLOUDSEARCH_ENDPOINT = "http://doc-cfa-healthtools-ke-doctors-m34xee6byjmzcgzmovevkjpffy.eu-west-1.cloudsearch.amazonaws.com/"
-cloudsearch_client = boto3.client("cloudsearchdomain",
-                                  endpoint_url=DOCTORS_CLOUDSEARCH_ENDPOINT,
-                                  **AWS_CONFIGS)
 
 
 @doctors_api.route('/', methods=['GET'])
@@ -46,13 +40,14 @@ def search():
 
         # get doctors by that name from aws
         response = {}
-        doctors = get_doctors_from_cloudsearch(query)
+        es = Elastic()
+        doctors = es.get_from_elasticsearch('doctors', query)
 
         if not doctors:
             response["message"] = "No doctor by that name found."
 
         track_event(current_app.config.get('GA_TRACKING_ID'), 'Doctor', 'search',
-                        request.remote_addr, label=query, value=len(doctors))
+                    request.remote_addr, label=query, value=len(doctors))
         response["data"] = {"doctors": doctors}
         response["status"] = "success"
 
@@ -64,11 +59,3 @@ def search():
             "message": str(err),
             "data": {"doctors": []}
         })
-
-
-def get_doctors_from_cloudsearch(query):
-    '''
-    Get doctors from AWS cloudsearch
-    '''
-    results = cloudsearch_client.search(query=query, size=10000)
-    return results["hits"]["hit"]

healthtools_ke_api/views/elastic_search.py

@@ -0,0 +1,63 @@
+from elasticsearch import Elasticsearch, RequestsHttpConnection
+from requests_aws4auth import AWS4Auth
+
+from healthtools_ke_api.settings import AWS, ES
+from serializer import JSONSerializerPython2
+import re
+
+
+class Elastic(object):
+    """
+    Common class for elastic search client and methods
+    """
+    def __init__(self):
+        # client host for aws elastic search service
+        if "aws" in ES["host"]:
+            # set up authentication credentials
+            awsauth = AWS4Auth(AWS["access_key"], AWS["secret_key"], AWS["region"], 'es')
+            self.es_client = Elasticsearch(
+                hosts=ES["host"],
+                port=int(ES["port"]),
+                http_auth=awsauth,
+                use_ssl=True,
+                verify_certs=True,
+                connection_class=RequestsHttpConnection,
+                serializer=JSONSerializerPython2()
+            )
+        else:
+            self.es_client = Elasticsearch("{}:{}".format(ES["host"], ES["port"]))
+
+    @staticmethod
+    def remove_keyword(query):
+        """
+        Remove keyword from search term
+        """
+        query_formatted = query.strip().lower()
+        keywords = ['dr', 'dr.', 'doctor', 'nurse', 'co', 'c.o.', 'c.o', 'clinical officer']
+        for word in keywords:
+            regex = r'(?<![\w\d]){0}(?![\w\d])'.format(word)
+            query_formatted = re.sub(regex, "", query_formatted)
+        return query_formatted.strip()
+
+    def get_from_elasticsearch(self, doc_type, query):
+        """
+        get data from elasticsearch
+        :return: Query results from elasticsearch
+        """
+        search_term = self.remove_keyword(query)
+        results = self.es_client.search(
+            index=ES['index'],
+            doc_type=doc_type,
+            body={
+                "query": {
+                    "match": {
+                        "name": {
+                            "query": search_term,
+                            "fuzziness": "auto",
+                            "prefix_length": 1
+                        }
+                    }
+                }
+            }
+        )
+        return results["hits"]["hits"]

healthtools_ke_api/views/nurses.py

@@ -7,9 +7,8 @@
 import requests
 import memcache
 
-
 nurses_api = Blueprint('nurses_api', __name__)
-cache = memcache.Client([(MEMCACHED_URL)], debug=True)  # cache server
+cache = memcache.Client([MEMCACHED_URL], debug=True)  # cache server
 
 nurse_fields = ["name", "licence_no", "valid_till"]
 NURSING_COUNCIL_URL = "http://nckenya.com/services/search.php?p=1&s={}"
@@ -45,7 +44,7 @@ def search():
                 "error": "A query is required.",
                 "results": "",
                 "data": {"nurses": []}
-            })
+                })
 
         # try to get queried result first
         cached_result = cache.get(query.replace(" ", ""))
@@ -98,7 +97,7 @@ def get_nurses_from_nc_registry(query):
 
     # make soup for parsing out of response and get the table
     soup = BeautifulSoup(response.content, "html.parser")
-    table = soup.find('table', {"class": "zebra"}).find("tbody")
+    table = soup.find("table", {"class": "zebra"}).find("tbody")
     rows = table.find_all("tr")
 
     # parse table for the nurses data

healthtools_ke_api/views/serializer.py

@@ -0,0 +1,18 @@
+import json
+from elasticsearch import serializer, compat, exceptions
+
+
+class JSONSerializerPython2(serializer.JSONSerializer):
+    """Override elasticsearch library serializer to ensure it encodes utf characters during json dump.
+    See original at: https://github.com/elastic/elasticsearch-py/blob/master/elasticsearch/serializer.py#L42
+    A description of how ensure_ascii encodes unicode characters to ensure they can be sent across the wire
+    as ascii can be found here: https://docs.python.org/2/library/json.html#basic-usage
+    """
+    def dumps(self, data):
+        # don't serialize strings
+        if isinstance(data, compat.string_types):
+            return data
+        try:
+            return json.dumps(data, default=self.default, ensure_ascii=True)
+        except (ValueError, TypeError) as e:
+            raise exceptions.SerializationError(data, e)