Merge pull request #279 from mapswipe/dev

v2.1.8

Author: Hagellach37

.travis

@@ -0,0 +1,19 @@
+# This is a special configuration file to run tests on Travis-CI via
+# GitHub notifications when changes are committed.
+#
+# For technical details, see http://travis-ci.org/
+
+language: python
+python:
+  - "3.6"
+  - "3.7"
+  - "3.8"
+
+before_install:
+ - pip install --upgrade pip setuptools
+ - pip install flake8 black
+ - flake8 --config=mapswipe_workers/setup.cfg mapswipe_workers/mapswipe_workers/*.py
+ - black --check mapswipe_workers/mapswipe_workers/*.py
+
+install:
+ - pip install mapswipe_workers/

docker-compose.yaml

@@ -53,7 +53,7 @@ services:
             context: mapswipe_workers/
         depends_on:
             - postgres
-        command: mapswipe_workers --verbose run --schedule m
+        command: mapswipe_workers --verbose run --schedule
         volumes:
             - ./api-data:/var/lib/mapswipe_workers/api-data/:rw
             - ./api-data/agg_results:/var/lib/mapswipe_workers/api-data/agg_results:rw

mapswipe_workers/.pre-commit-config.yaml

@@ -7,4 +7,3 @@ repos:
     rev: 3.7.9
     hooks:
       - id: flake8
-        args: ["--ignore=E501,W503"]

mapswipe_workers/Dockerfile

@@ -1,12 +1,15 @@
-# This image contains Python 3 and the latest pre-compiled version of GDAL
-# Based on Debian (Including apt-get update && upgrade)
-FROM thinkwhere/gdal-python:3.7-shippable
+# https://github.com/OSGeo/gdal/tree/master/gdal/docker
+# Image includes python3.6, gdal-python, gdal-bin
+FROM osgeo/gdal:ubuntu-small-latest
 
-# Install gdal-bin to get ogr2ogr tool
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+
+# Install pip
 RUN apt-get update
-RUN apt-get --yes install gdal-bin
+RUN apt-get --yes install python3-pip
 
-# create directories for config, logs and data
+# Create directories for config, logs and data
 ARG config_dir=/usr/share/config/mapswipe_workers/
 ARG repo_dir=/usr/local/mapswipe_workers/
 ARG data_dir=/var/lib/mapswipe_workers/
@@ -26,7 +29,7 @@ RUN mkdir -p $data_dir"api-data/tasks/"
 RUN mkdir -p $data_dir"api-data/yes_maybe/"
 RUN mkdir -p $data_dir"api-data/hot_tm/"
 
-# copy mapswipe workers repo from local repo
+# Copy mapswipe workers repo from local repo
 WORKDIR $repo_dir
 COPY mapswipe_workers/ mapswipe_workers/
 COPY sample_data/ sample_data/
@@ -35,8 +38,8 @@ COPY requirements.txt .
 COPY setup.py .
 COPY config $config_dir
 
-# Install dependencies and mapswipe-workers
-# RUN python setup.py install
-RUN pip install .
+# Update setuptools and install mapswipe-workers with dependencies (requirements.txt)
+RUN pip3 install --upgrade setuptools
+RUN pip3 install .
 
-# we don't use a CMD here, this will be defined in docker-compose.yaml
+# Don't use a CMD here, this will be defined in docker-compose.yaml

mapswipe_workers/config/example-configuration.json

@@ -26,8 +26,7 @@
     },
     "slack": {
         "token": "your_slack_token",
-        "channel": "your_slack_channel",
-        "username": "your_slack_username"
+        "channel": "your_slack_channel"
     },
     "sentry": {
         "dsn": "your_sentry_dsn_value"

mapswipe_workers/config/logging.cfg

@@ -30,4 +30,4 @@ formatter=mapswipeFormatter
 args=('/var/log/mapswipe_workers/mapswipe_workers.log','D', 1, 14,)
 
 [formatter_mapswipeFormatter]
-format=%(asctime)s - %(name)s - %(levelname)s - %(message)s
+format=%(asctime)s - %(levelname)s - %(filename)s - %(funcName)s - %(message)s

mapswipe_workers/mapswipe_workers/auth.py

@@ -1,150 +1,74 @@
-#!/usr/bin/python3
-#
-# Author: B. Herfort, M. Reinmuth, 2017
-############################################
-
-import json
-
-import psycopg2
 import firebase_admin
-from firebase_admin import credentials
-from firebase_admin import db
-
-from mapswipe_workers.definitions import CONFIG_PATH
-from mapswipe_workers.definitions import SERVICE_ACCOUNT_KEY_PATH
+import psycopg2
+from firebase_admin import credentials, db
 
+from mapswipe_workers.definitions import CONFIG, SERVICE_ACCOUNT_KEY_PATH
 
-def load_config():
-    """
-    Loads the user configuration values.
 
-    Returns
-    -------
-    dictonary
-    """
-    with open(CONFIG_PATH) as f:
-        CONFIG = json.load(f)
-    return CONFIG
+def get_api_key(tileserver: str) -> str:
+    if tileserver == "custom":
+        return None
+    else:
+        return CONFIG["imagery"][tileserver]["api_key"]
 
 
-def get_api_key(tileserver):
-    CONFIG = load_config()
-    try:
-        if tileserver == 'custom':
-            return None
-        else:
-            return CONFIG['imagery'][tileserver]['api_key']
-    except KeyError:
-        print(
-                f'Could not find the API key for imagery tileserver '
-                f'{tileserver} in {CONFIG_PATH}.'
-                )
-        raise
-
-
-def get_tileserver_url(tileserver):
-    CONFIG = load_config()
-    try:
-        if tileserver == 'custom':
-            return None
-        else:
-            return CONFIG['imagery'][tileserver]['url']
-    except KeyError:
-        print('Could not find the url for imagery tileserver {} in {}.'.format(
-            tileserver,
-            CONFIG_PATH
-            ))
-        raise
-
-
-def init_firebase():
-    try:
-        # Is an App instance already initialized?
-        firebase_admin.get_app()
-    except ValueError:
-        cred = credentials.Certificate(SERVICE_ACCOUNT_KEY_PATH)
-        # Initialize the app with a service account, granting admin privileges
-        firebase_admin.initialize_app(cred)
+def get_tileserver_url(tileserver: str) -> str:
+    if tileserver == "custom":
+        return None
+    else:
+        return CONFIG["imagery"][tileserver]["url"]
 
 
-def firebaseDB():
+def firebaseDB() -> object:
     try:
         # Is an App instance already initialized?
         firebase_admin.get_app()
         # Return the imported Firebase Realtime Database module
         return db
     except ValueError:
         cred = credentials.Certificate(SERVICE_ACCOUNT_KEY_PATH)
-        config = load_config()
-        databaseName = config['firebase']['database_name']
-        databaseURL = f'https://{databaseName}.firebaseio.com'
+        databaseName = CONFIG["firebase"]["database_name"]
+        databaseURL = f"https://{databaseName}.firebaseio.com"
 
         # Initialize the app with a service account, granting admin privileges
-        firebase_admin.initialize_app(cred, {
-            'databaseURL': databaseURL
-        })
+        firebase_admin.initialize_app(cred, {"databaseURL": databaseURL})
 
         # Return the imported Firebase Realtime Database module
         return db
 
 
 class postgresDB(object):
+    """Helper calss for Postgres interactions"""
+
     _db_connection = None
     _db_cur = None
 
     def __init__(self):
-        CONFIG = load_config()
-        try:
-            host = CONFIG['postgres']['host']
-            port = CONFIG['postgres']['port']
-            dbname = CONFIG['postgres']['database']
-            user = CONFIG['postgres']['username']
-            password = CONFIG['postgres']['password']
-        except KeyError:
-            raise Exception(
-                    f'Could not load postgres credentials '
-                    f'from the configuration file'
-                    )
+        host = CONFIG["postgres"]["host"]
+        port = CONFIG["postgres"]["port"]
+        dbname = CONFIG["postgres"]["database"]
+        user = CONFIG["postgres"]["username"]
+        password = CONFIG["postgres"]["password"]
 
         self._db_connection = psycopg2.connect(
-                database=dbname,
-                user=user,
-                password=password,
-                host=host,
-                port=port
-                )
+            database=dbname, user=user, password=password, host=host, port=port,
+        )
 
     def query(self, query, data=None):
         self._db_cur = self._db_connection.cursor()
         self._db_cur.execute(query, data)
         self._db_connection.commit()
         self._db_cur.close()
 
-    def copy_from(
-            self,
-            f,
-            table,
-            columns
-            ):
+    def copy_from(self, f, table, columns):
         self._db_cur = self._db_connection.cursor()
-        self._db_cur.copy_from(
-                f,
-                table,
-                columns=columns
-                )
+        self._db_cur.copy_from(f, table, columns=columns)
         self._db_connection.commit()
         self._db_cur.close()
 
-    def copy_expert(
-            self,
-            sql,
-            file,
-            ):
+    def copy_expert(self, sql, file):
         self._db_cur = self._db_connection.cursor()
-        self._db_cur.copy_expert(
-                sql,
-                file,
-                )
+        self._db_cur.copy_expert(sql, file)
         self._db_connection.commit()
         self._db_cur.close()
 

mapswipe_workers/mapswipe_workers/base/base_project.py

@@ -78,7 +78,7 @@ def __init__(self, project_draft):
     # def resultRequiredCounter(self):
     #     return self.resultRequiredCounter
 
-    def save_project(self, fb_db):
+    def save_project(self):
         """
         Creates a projects with groups and tasks
         and saves it in firebase and postgres
@@ -130,7 +130,6 @@ def save_project(self, fb_db):
                     )
             try:
                 self.save_to_firebase(
-                        fb_db,
                         project,
                         groups,
                         groupsOfTasks,
@@ -160,7 +159,7 @@ def save_project(self, fb_db):
                     )
             raise CustomError(e)
 
-    def save_to_firebase(self, fb_db, project, groups, groupsOfTasks):
+    def save_to_firebase(self, project, groups, groupsOfTasks):
 
         # remove wkt geometry attribute of projects and tasks
         project.pop('geometry', None)
@@ -169,6 +168,7 @@ def save_to_firebase(self, fb_db, project, groups, groupsOfTasks):
                 groupsOfTasks[group_id][i].pop('geometry', None)
 
 
+        fb_db = auth.firebaseDB()
         ref = fb_db.reference('')
         # save project
         ref.update({

mapswipe_workers/mapswipe_workers/definitions.py

@@ -1,41 +1,37 @@
-import os
 import json
 import logging
 import logging.config
-from logging.handlers import TimedRotatingFileHandler
+import os
+
+import sentry_sdk
+
+
+class CustomError(Exception):
+    pass
+
+
+def load_config(CONFIG_PATH) -> dict:
+    """Read the configuration file."""
+    with open(CONFIG_PATH) as f:
+        return json.load(f)
 
 
 ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
 
-CONFIG_DIR = os.path.abspath(
-        '/usr/share/config/mapswipe_workers/'
-        )
+CONFIG_DIR = os.path.abspath("/usr/share/config/mapswipe_workers/")
 
-CONFIG_PATH = os.path.join(
-        CONFIG_DIR,
-        'configuration.json'
-        )
+CONFIG_PATH = os.path.join(CONFIG_DIR, "configuration.json")
 
-SERVICE_ACCOUNT_KEY_PATH = os.path.join(
-        CONFIG_DIR,
-        'serviceAccountKey.json'
-        )
+CONFIG = load_config(CONFIG_PATH)
 
-LOGGING_CONFIG_PATH = os.path.join(
-        CONFIG_DIR,
-        'logging.cfg'
-        )
+SERVICE_ACCOUNT_KEY_PATH = os.path.join(CONFIG_DIR, "serviceAccountKey.json")
 
-DATA_PATH = os.path.abspath(
-        '/var/lib/mapswipe_workers/'
-        )
+LOGGING_CONFIG_PATH = os.path.join(CONFIG_DIR, "logging.cfg")
 
-logging.config.fileConfig(
-        fname=LOGGING_CONFIG_PATH,
-        disable_existing_loggers=True
-        )
-logger = logging.getLogger('Mapswipe Workers')
+DATA_PATH = os.path.abspath("/var/lib/mapswipe_workers/")
 
+logging.config.fileConfig(fname=LOGGING_CONFIG_PATH, disable_existing_loggers=True)
+logger = logging.getLogger("Mapswipe Workers")
 
-class CustomError(Exception):
-    pass
+sentry_sdk.init(CONFIG["sentry"]["dsn"])
+sentry = sentry_sdk