Merge pull request #24 from mapswipe/eric.p.boucher-fix-imports

Hagellach37 · web-flow · commit 152aa4f727fa · 2018-05-26T12:54:36.000+02:00
Fix imports for osgeo and utils
diff --git a/.travis.yml b/.travis.yml
@@ -6,6 +6,7 @@ python:
 before_install:
   # install gdal
   - sudo bash install_gdal.sh
+  - cp ./cfg/your_config_file.cfg ./cfg/config.cfg
 install:
   - export CPLUS_INCLUDE_PATH=/usr/include/gdal
   - export C_INCLUDE_PATH=/usr/include/gdal
diff --git a/cfg/auth.py b/cfg/auth.py
@@ -8,7 +8,7 @@
 import json
 import sys
 
-CONFIG_PATH = '../cfg/config.cfg'
+CONFIG_PATH = './cfg/config.cfg'
 
 try:
     with open(CONFIG_PATH) as f:
@@ -35,7 +35,7 @@ def firebase_admin_auth():
     except:
         # Default Configuration
         print('could not get firebase informtaion from config file')
-        sys.exit()
+        sys.exit(1)
 
     # adapt this to your firebase setting
     config = {
@@ -59,8 +59,8 @@ def dev_firebase_admin_auth():
         service_account = CONFIG['dev_firebase']['service_account']
     except:
         # Default Configuration
-        print('could not get firebase dev information from config file')
-        sys.exit()
+        print('Could not get firebase dev information from config file.')
+        sys.exit(1)
 
     # adapt this to your firebase setting
     config = {
@@ -83,7 +83,7 @@ def get_api_key(tileserver):
     try:
         return CONFIG['imagery'][tileserver]
     except KeyError:
-        print("Couldn't find the API key for imagery tileserver {} in {}".format(tileserver, CONFIG_PATH))
+        print('Could not find the API key for imagery tileserver {} in {}.'.format(tileserver, CONFIG_PATH))
         raise
 
 
@@ -109,8 +109,8 @@ def __init__(self):
             host = CONFIG['mysql']['host']
             # print('use configuration for mysql as provided by config.json')
         except:
-            print('we could not load mysql info the config file')
-            sys.exit()
+            print('Could not load mysql info from config file.')
+            sys.exit(1)
 
         self._db_connection = pymysql.connect(
             database=dbname,
@@ -155,7 +155,7 @@ def __init__(self):
         except:
             # Default configuration
             print('we could not load mysql dev info from the config file')
-            sys.exit()
+            sys.exit(1)
 
         self._db_connection = pymysql.connect(
             database=dbname,
diff --git a/cfg/loop_export_module.json b/cfg/loop_export_module.json
@@ -1,7 +1,7 @@
 {
   "name": "loop_export_module",
-  "script": "run_export.py",
-  "cwd": "/data/python-mapswipe-workers/export_module",
+  "script": "./export_module/run_export.py",
+  "cwd": "/data/python-mapswipe-workers/",
   "exec_interpreter": "/data/environments/mapswipe_workers/bin/python3",
   "log_date_format": "YYYY-MM-DD HH:mm Z",
   "autorestart": true,
diff --git a/cfg/loop_import_module.json b/cfg/loop_import_module.json
@@ -1,7 +1,7 @@
 {
   "name": "loop_import_module",
-  "script": "run_import.py",
-  "cwd": "/data/python-mapswipe-workers/import_module",
+  "script": "./import_module/run_import.py",
+  "cwd": "/data/python-mapswipe-workers/",
   "exec_interpreter": "/data/environments/mapswipe_workers/bin/python3",
   "log_date_format": "YYYY-MM-DD HH:mm Z",
   "autorestart": true,
diff --git a/cfg/loop_transfer_results.json b/cfg/loop_transfer_results.json
@@ -1,9 +1,9 @@
 {
   "name": "loop_transfer_results_module",
-  "script": "transfer_results.py",
-  "cwd": "/data/python-mapswipe-workers/transfer_results_module",
+  "script": "./transfer_results_module/transfer_results.py",
+  "cwd": "/data/python-mapswipe-workers",
   "exec_interpreter": "/data/environments/mapswipe_workers/bin/python3",
   "log_date_format": "YYYY-MM-DD HH:mm Z",
   "autorestart": true,
   "args": ["-l", "-m=1", "-s=60"]
-}
+}
diff --git a/cfg/loop_update_module.json b/cfg/loop_update_module.json
@@ -1,7 +1,7 @@
 {
   "name": "loop_update_module",
-  "script": "run_update.py",
-  "cwd": "/data/python-mapswipe-workers/update_module",
+  "script": "./update_module/run_update.py",
+  "cwd": "/data/python-mapswipe-workers",
   "exec_interpreter": "/data/environments/mapswipe_workers/bin/python3",
   "log_date_format": "YYYY-MM-DD HH:mm Z",
   "autorestart": true,
diff --git a/cfg/your_config_file.cfg b/cfg/your_config_file.cfg
@@ -10,7 +10,7 @@
   "auth_domain": "your_firebase_auth_domain",
   "database_url": "your_firebase_database_url",
   "storage_bucket": "your_firebase_storage_bucket",
-  "service_account": "your_firebase_service_account_file"
+  "service_account": "./cfg/your_firebase_service_account_file"
   },
 "imagery":{
   "bing": "your_bing_api_key",
diff --git a/export_module/export_project_results.py b/export_module/export_project_results.py
@@ -13,8 +13,8 @@
 
 import logging
 import time
-from auth import firebase_admin_auth
-from auth import mysqlDB
+from cfg.auth import firebase_admin_auth
+from cfg.auth import mysqlDB
 
 import argparse
 # define arguments that can be passed by the user
@@ -62,23 +62,23 @@ def get_project_results(project_id):
           ,task_y
           ,task_z
           ,avg(result) as decision
-          ,SUM(CASE 
+          ,SUM(CASE
             WHEN result = 1 THEN 1
             ELSE 0
            END) AS yes_count
-           ,SUM(CASE 
+           ,SUM(CASE
             WHEN result = 2 THEN 1
             ELSE 0
            END) AS maybe_count
-           ,SUM(CASE 
+           ,SUM(CASE
             WHEN result = 3 THEN 1
             ELSE 0
            END) AS bad_imagery_count
            ,wkt
         from
           results
         where
-          project_id = %s and result > 0 
+          project_id = %s and result > 0
         group by
           task_id, wkt'''
 
diff --git a/export_module/export_projects.py b/export_module/export_projects.py
@@ -13,7 +13,7 @@
 
 import logging
 import time
-from auth import firebase_admin_auth
+from cfg.auth import firebase_admin_auth
 
 import argparse
 # define arguments that can be passed by the user
diff --git a/export_module/export_users_and_stats.py b/export_module/export_users_and_stats.py
@@ -13,7 +13,7 @@
 
 import logging
 import time
-from auth import firebase_admin_auth
+from cfg.auth import firebase_admin_auth
 
 import argparse
 # define arguments that can be passed by the user
diff --git a/export_module/run_export.py b/export_module/run_export.py
@@ -11,7 +11,8 @@
 from export_module.export_projects import export_projects
 from export_module.export_users_and_stats import export_users_and_stats
 from cfg.auth import firebase_admin_auth
-from utils.send_slack_message import send_slack_message
+
+from utils import error_handling
 
 import time
 import argparse
diff --git a/import_module/create_groups.py b/import_module/create_groups.py
@@ -2,18 +2,16 @@
 # -*- coding: UTF-8 -*-
 # Author: M. Reinmuth, B. Herfort
 ####################################################################################################
-
+import os
 import sys
 
 import json
 import logging
 import math
-import os
-import ogr
-import osr
-import gdal
 
-from auth import get_api_key
+from osgeo import osr, ogr, gdal
+
+from cfg.auth import get_api_key
 
 import argparse
 
diff --git a/import_module/run_import.py b/import_module/run_import.py
@@ -4,22 +4,23 @@
 ########################################################################################################################
 
 import sys
+import time
 # add some files in different folders to sys.
 # these files can than be loaded directly
 sys.path.insert(0, '../cfg/')
 sys.path.insert(0, '../utils/')
 
 import logging
 import os
-import time
-import ogr
 
-from auth import get_submission_key
-from create_groups import run_create_groups
+from osgeo import ogr
+
+from cfg.auth import get_submission_key
+from import_module.create_groups import run_create_groups
 
-import error_handling
+from utils import error_handling
 
-from send_slack_message import send_slack_message
+from utils.send_slack_message import send_slack_message
 
 import argparse
 parser = argparse.ArgumentParser(description='Process some integers.')
@@ -441,13 +442,13 @@ def run_import():
 
     if args.modus == 'development':
         # we use the dev instance for testing
-        from auth import dev_firebase_admin_auth as firebase_admin_auth
-        from auth import dev_mysqlDB as mysqlDB
+        from cfg.auth import dev_firebase_admin_auth as firebase_admin_auth
+        from cfg.auth import dev_mysqlDB as mysqlDB
         print('We are using the development instance')
     elif args.modus == 'production':
         # we use the dev instance for testing
-        from auth import firebase_admin_auth as firebase_admin_auth
-        from auth import mysqlDB as mysqlDB
+        from cfg.auth import firebase_admin_auth as firebase_admin_auth
+        from cfg.auth import mysqlDB as mysqlDB
         print('We are using the production instance')
 
     # check whether arguments are correct
diff --git a/install_gdal.sh b/install_gdal.sh
@@ -5,7 +5,8 @@ apt-get install -y build-essential
 apt-get install -y gcc
 apt-get install -y gdal-bin
 apt-get install -y python3-dev
-apt-get install -y libgdal1-dev libgdal-dev
+apt-get install -y -q libgdal1-dev
+apt-get install -y -q libgdal-dev
 export CPLUS_INCLUDE_PATH=/usr/include/gdal
 export C_INCLUDE_PATH=/usr/include/gdal
 apt-get install -y libgdal1h
diff --git a/test/test_export_modules.py b/test/test_export_modules.py
@@ -0,0 +1,6 @@
+"""Test export modules."""
+from export_module import export_project_results, export_projects, export_users_and_stats
+
+
+def test_export_module():
+	pass
diff --git a/test/test_import_module.py b/test/test_import_module.py
@@ -0,0 +1,6 @@
+"""Basic tests for the import module."""
+from import_module import create_groups, run_import
+
+
+def test_import_module():
+	pass
diff --git a/transfer_results_module/transfer_results.py b/transfer_results_module/transfer_results.py
@@ -2,8 +2,6 @@
 # -*- coding: UTF-8 -*-
 # Author: M. Reinmuth, B. Herfort
 ####################################################################################################
-
-import error_handling
 import logging
 import json
 import os
@@ -14,6 +12,8 @@
 from cfg.auth import firebase_admin_auth
 from cfg.auth import mysqlDB
 
+from utils import error_handling
+
 import argparse
 
 parser = argparse.ArgumentParser(description='Process some integers.')
diff --git a/update_module/run_update.py b/update_module/run_update.py
@@ -6,8 +6,9 @@
 import logging
 import sys
 
-from update_modue.update_project_contributors import update_project_contributors
-from update_modue.update_project_progress import update_project_progress
+from update_module.update_project_contributors import update_project_contributors
+from update_module.update_project_progress import update_project_progress
+
 from cfg.auth import firebase_admin_auth
 from utils.send_slack_message import send_slack_message
 
diff --git a/update_module/update_project_contributors.py b/update_module/update_project_contributors.py
@@ -10,17 +10,11 @@
 sys.path.insert(0, '../utils/')
 
 import logging
-import numpy as np
 import os
-import threading
 import time
-from queue import Queue
-
-import requests
-
-from auth import firebase_admin_auth
-from auth import mysqlDB
 
+from cfg.auth import firebase_admin_auth
+from cfg.auth import mysqlDB
 
 
 import argparse
diff --git a/update_module/update_project_progress.py b/update_module/update_project_progress.py
@@ -16,7 +16,7 @@
 import time
 from queue import Queue
 import requests
-from auth import firebase_admin_auth
+from cfg.auth import firebase_admin_auth
 import argparse
 
 
diff --git a/utils/error_handling.py b/utils/error_handling.py
@@ -4,7 +4,7 @@
 
 import logging
 
-from send_slack_message import send_slack_message
+from utils.send_slack_message import send_slack_message
 
 
 def _get_error_message_details(error):
