Merge pull request #148 from c-simpson/test_harness

Initial changes to support testing

Author: urirot

src/server/Dockerfile

@@ -15,12 +15,14 @@ RUN pip install --no-cache-dir -r /requirements.txt
 COPY . .
 
 RUN mkdir /app/static \
-          /app/static/raw_data \
-          /app/static/raw_data/current \
-          /app/static/output \
-          /app/static/output/reports \
-          /app/static/logs \
-          /app/static/zipped
+    /app/static/raw_data \
+    /app/static/raw_data/current \
+    /app/static/output \
+    /app/static/output/reports \
+    /app/static/logs \
+    /app/static/zipped
+
+RUN [ ! -f /app/static/logs/last_execution.json ] && printf null > /app/static/logs/last_execution.json
 
 RUN chmod -R 777 /app/static
 

src/server/api/admin_api.py

@@ -8,21 +8,28 @@
 from config import engine
 from flask import send_file, request, redirect, jsonify, current_app, abort
 from api.file_uploader import validate_and_arrange_upload
-from config import RAW_DATA_PATH, OUTPUT_PATH, CURRENT_SOURCE_FILES_PATH, ZIPPED_FILES, LOGS_PATH
+from config import (
+    RAW_DATA_PATH,
+    OUTPUT_PATH,
+    CURRENT_SOURCE_FILES_PATH,
+    ZIPPED_FILES,
+    LOGS_PATH,
+)
+
+ALLOWED_EXTENSIONS = {"csv", "xlsx"}
 
-ALLOWED_EXTENSIONS = {'csv', 'xlsx'}
 
 def __allowed_file(filename):
-    return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
+    return "." in filename and filename.rsplit(".", 1)[1].lower() in ALLOWED_EXTENSIONS
 
 
 # file upload tutorial
-@admin_api.route('/api/file', methods=['POST'])
+@admin_api.route("/api/file", methods=["POST"])
 def uploadCSV():
-    if 'file' not in request.files:
+    if "file" not in request.files:
         return redirect(request.url)
 
-    for file in request.files.getlist('file'):
+    for file in request.files.getlist("file"):
         if __allowed_file(file.filename):
             try:
                 validate_and_arrange_upload(file, RAW_DATA_PATH)
@@ -31,34 +38,39 @@ def uploadCSV():
             finally:
                 file.close()
 
-    return redirect('/')
+    return redirect("/")
 
 
-@admin_api.route('/api/files/<destination>', methods=['GET'])
+@admin_api.route("/api/files/<destination>", methods=["GET"])
 def files(destination):
-    current_app.logger.info('Start returning zip of all data')
-    if request.args.get('download_current_btn'):
+    current_app.logger.info("Start returning zip of all data")
+    if request.args.get("download_current_btn"):
         source = RAW_DATA_PATH + destination
-    if request.args.get('download_archived_btn'):
+    if request.args.get("download_archived_btn"):
         source = RAW_DATA_PATH
-    if request.args.get('download_output_btn'):
+    if request.args.get("download_output_btn"):
         source = OUTPUT_PATH
 
-    zip_name = destination + '_data_out'
+    zip_name = destination + "_data_out"
 
     try:
-        current_app.logger.info(shutil.make_archive(ZIPPED_FILES + zip_name, 'zip', source))
-        return send_file(ZIPPED_FILES + zip_name + '.zip', as_attachment=True,
-                         attachment_filename=zip_name + '.zip')
+        current_app.logger.info(
+            shutil.make_archive(ZIPPED_FILES + zip_name, "zip", source)
+        )
+        return send_file(
+            ZIPPED_FILES + zip_name + ".zip",
+            as_attachment=True,
+            attachment_filename=zip_name + ".zip",
+        )
     except Exception as e:
         return str(e)
 
 
-@admin_api.route('/api/listCurrentFiles', methods=['GET'])
+@admin_api.route("/api/listCurrentFiles", methods=["GET"])
 def listCurrentFiles():
     result = None
 
-    current_app.logger.info('Start returning file list')
+    current_app.logger.info("Start returning file list")
     file_list_result = os.listdir(CURRENT_SOURCE_FILES_PATH)
 
     if len(file_list_result) > 0:
@@ -67,27 +79,24 @@ def listCurrentFiles():
     return jsonify(result)
 
 
-@admin_api.route('/api/execute', methods=['GET'])
+@admin_api.route("/api/execute", methods=["GET"])
 def execute():
-    current_app.logger.info('Execute flow')
+    current_app.logger.info("Execute flow")
     flow_script.start_flow()
 
     current_time = datetime.now().ctime()
     statistics = getStatistics()
 
-    last_execution_details = {
-        "executionTime": current_time,
-        "stats": statistics
-    }
+    last_execution_details = {"executionTime": current_time, "stats": statistics}
 
-    last_execution_file = open(LOGS_PATH + 'last_execution.json', 'w')
+    last_execution_file = open(LOGS_PATH + "last_execution.json", "w")
     last_execution_file.write(json.dumps(last_execution_details))
     last_execution_file.close()
 
     return jsonify(success=True)
 
 
-'''
+"""
 @admin_api.route('/api/status', methods=['GET'])
 def checkStatus():
     with engine.connect() as connection:
@@ -99,33 +108,46 @@ def checkStatus():
         for row in query_result:
             results = dict(row)
         return jsonify(results)
-'''
+"""
 
 
-@admin_api.route('/api/statistics', methods=['GET'])
+@admin_api.route("/api/statistics", methods=["GET"])
 def listStatistics():
     try:
-        last_execution_file = open(LOGS_PATH + 'last_execution.json', 'r')
+        last_execution_file = open(LOGS_PATH + "last_execution.json", "r")
         last_execution_details = json.loads(last_execution_file.read())
         last_execution_file.close()
 
+    except (FileNotFoundError):
+        current_app.logger.error("last_execution.json file was missing")
+        return abort(500)
+
+    except (json.JSONDecodeError):
+        current_app.logger.error(
+            "last_execution.json could not be decoded - possible corruption"
+        )
+        return abort(500)
+
     except Exception as e:
-        return abort(404)
+        current_app.logger.error("Failure reading last_execution.json: ", e)
+        return abort(500)
 
     return jsonify(last_execution_details)
 
 
 def getStatistics():
     with engine.connect() as connection:
-        query = text("SELECT \
-            SUM(CASE WHEN salesforcecontacts_id is not null and volgistics_id is null and shelterluvpeople_id is null THEN 1 ELSE 0 END) AS \"Only SalesForce Contacts\", \
-            SUM(CASE WHEN volgistics_id is not null and shelterluvpeople_id is null and salesforcecontacts_id is null THEN 1 ELSE 0 END) AS \"Only Volgistics Contacts\", \
-            SUM(CASE WHEN shelterluvpeople_id is not null and volgistics_id is null and salesforcecontacts_id is null THEN 1 ELSE 0 END) AS \"Only Shelterluv Contacts\", \
-            SUM(CASE WHEN salesforcecontacts_id is not null and shelterluvpeople_id is not null and volgistics_id is null THEN 1 ELSE 0 END) AS \"Only Salesforcec & Shelterluv\", \
-            SUM(CASE WHEN salesforcecontacts_id is not null and volgistics_id is not null and shelterluvpeople_id is null THEN 1 ELSE 0 END) AS \"Only Salesforce & Volgistics\", \
-            SUM(CASE WHEN volgistics_id is not null and shelterluvpeople_id is not null and salesforcecontacts_id is null THEN 1 ELSE 0 END) AS \"Only Shelterluv & Volgistics\", \
-            SUM(CASE WHEN salesforcecontacts_id is not null and volgistics_id is not null and shelterluvpeople_id is not null THEN 1 ELSE 0 END) AS \"Salesforcec & Shelterluv & Volgistics\" \
-            FROM master")
+        query = text(
+            'SELECT \
+            SUM(CASE WHEN salesforcecontacts_id is not null and volgistics_id is null and shelterluvpeople_id is null THEN 1 ELSE 0 END) AS "Only SalesForce Contacts", \
+            SUM(CASE WHEN volgistics_id is not null and shelterluvpeople_id is null and salesforcecontacts_id is null THEN 1 ELSE 0 END) AS "Only Volgistics Contacts", \
+            SUM(CASE WHEN shelterluvpeople_id is not null and volgistics_id is null and salesforcecontacts_id is null THEN 1 ELSE 0 END) AS "Only Shelterluv Contacts", \
+            SUM(CASE WHEN salesforcecontacts_id is not null and shelterluvpeople_id is not null and volgistics_id is null THEN 1 ELSE 0 END) AS "Only Salesforcec & Shelterluv", \
+            SUM(CASE WHEN salesforcecontacts_id is not null and volgistics_id is not null and shelterluvpeople_id is null THEN 1 ELSE 0 END) AS "Only Salesforce & Volgistics", \
+            SUM(CASE WHEN volgistics_id is not null and shelterluvpeople_id is not null and salesforcecontacts_id is null THEN 1 ELSE 0 END) AS "Only Shelterluv & Volgistics", \
+            SUM(CASE WHEN salesforcecontacts_id is not null and volgistics_id is not null and shelterluvpeople_id is not null THEN 1 ELSE 0 END) AS "Salesforcec & Shelterluv & Volgistics" \
+            FROM master'
+        )
         query_result = connection.execute(query)
 
         # Need to iterate over the results proxy

src/server/requirements.txt

@@ -8,4 +8,5 @@ psycopg2-binary==2.8.4
 python-Levenshtein-wheels
 xlrd
 openpyxl
-requests
+requests
+pytest

src/server/test_api.py

@@ -0,0 +1,52 @@
+import pytest, socket, requests
+
+#
+# Run 'pytest' from the command line
+#
+# Running pytest can result in six different exit codes:
+#   0 - All tests were collected and passed successfully
+#   1 - Tests were collected and run but some of the tests failed
+#   2 - Test execution was interrupted by the user
+#   3 - Internal error happened while executing tests
+#   4 - pytest command line usage error
+#   5 - No tests were collected
+#
+# These codes are represented by the pytest.ExitCode enum
+
+
+SERVER_URL = "http://server:5000"
+
+###  DNS lookup tests
+
+# Ensure DNS not resolving bad host names
+def test_bad_dns():
+    with pytest.raises(socket.gaierror):
+        socket.getaddrinfo("bad_server_name_that_should_not_resolve", "5000")
+
+
+# Do we get IPs for good names?
+def test_db_dns():
+    assert (
+        len(socket.getaddrinfo("db", "5000")) > 0
+    )  # getaddrinfo works for IPv4 and v6
+
+
+def test_server_dns():
+    assert len(socket.getaddrinfo("server", "5000")) > 0
+
+
+def test_client_dns():
+    assert len(socket.getaddrinfo("client", "5000")) > 0
+
+
+# Simple API tests
+
+def test_currentFiles():
+    response = requests.get(SERVER_URL + "/api/listCurrentFiles")
+    assert response.status_code == 200
+
+
+def test_statistics():
+    response = requests.get(SERVER_URL + "/api/statistics")
+    assert response.status_code == 200
+