Merge pull request #340 from mapswipe/dev

All configuration is now stored in environment vars.

Author: Matthias-Schaub

.travis

@@ -0,0 +1,71 @@
+# This is a special configuration file to run tests on Travis-CI via
+# GitHub notifications when changes are committed.
+#
+# For technical details, see http://travis-ci.org/
+#
+branches:
+  except:
+    - docs
+
+dist: bionic
+addons:
+  apt:
+    sources:
+      - ppa:ubuntugis/ppa
+    update: true
+    packages:
+      - gdal-bin
+      - python-gdal
+      - python3-gdal
+      - libgdal-dev
+
+language: python
+python:
+  - "3.6"
+virtualenv:
+  system_site_packages: true
+
+services:
+  - docker
+
+before_install:
+  - pip install --upgrade pip setuptools
+  - pip install flake8 black
+
+install:
+  - pip install mapswipe_workers/
+  - mkdir --parents ~/.config/mapswipe_workers
+  - mkdir --parents ~/.local/share/mapswipe_workers
+  - echo "$SERVICE_ACCOUNT_KEY" > ~/.config/mapswipe_workers/serviceAccountKey.json
+  - docker build --tag mapswipe_postgres --file postgres/Dockerfile-dev postgres/
+
+before_script:
+  - docker run --detach --publish 5432:5432 --name mapswipe_postgres -e POSTGRES_DB="$POSTGRES_DB" -e POSTGRES_USER="$POSTGRES_USER" -e POSTGRES_PASSWORD="$POSTGRES_PASSWORD" mapswipe_postgres
+  - docker-compose up --build --detach firebase_deploy
+
+script:
+  # - TODO: Run black and flake8
+  - python -m unittest -v tests.integration_tests.test_gdal
+  # TODO: Use line below once tests have been implemented
+  #- python -m unittest discover -v -s mapswipe_workers/tests/integration_tests/ -p 'test_*.py'
+
+# before_deploy:
+#   # SSH setup to deploy to server after build.
+#   - apk add openssh-client
+#   - mkdir -p ~/.ssh
+#   - chmod 700 ~/.ssh
+#   - eval "$(ssh-agent -s)"
+#   - export KEY_FILE=~/.ssh/api_server_ssh_key
+#   - echo "$DEPLOYMENT_SERVER_PRIVATE_KEY" | tr -d '\r' > $KEY_FILE
+#   - chmod 600 $KEY_FILE
+#   - ssh-add $KEY_FILE > /dev/null
+#   - ssh-keyscan -H "$DEPLOYMENT_SERVER_IP" >> ~/.ssh/known_hosts
+#   - chmod 644 ~/.ssh/known_hosts
+#   - ssh -i $KEY_FILE -o StrictHostKeyChecking=no "mapswipe@$DEPLOYMENT_SERVER_IP"
+
+# deploy:
+#   provider: script
+#   script: bash scripts/deploy.sh
+#   on:
+#     branch: dev
+

.travis.yml

@@ -51,6 +51,26 @@ services:
         container_name: mapswipe_workers
         build:
             context: mapswipe_workers/
+        environment:
+            FIREBASE_DB: '${FIREBASE_DB}'
+            FIREBASE_API_KEY: '{FIREBASE_API_KEY}'
+            FIREBASE_TOKEN: '{FIREBASE_TOKEN}'
+            GOOGLE_APPLICATION_CREDENTIALS: 'serviceAccountKey.json'
+            POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}'
+            POSTGRES_USER: '${POSTGRES_USER}'
+            POSTGRES_DB: '${POSTGRES_DB}'
+            POSTGRES_HOST: 'postgres'
+            POSTGRES_PORT: 5432
+            PGDATA: '/var/lib/postgresql/mapswipe'
+            IMAGE_BING_API_KEY: '${IMAGE_BING_API_KEY}'
+            IMAGE_ESRI_API_KEY: '${IMAGE_ESRI_API_KEY}'
+            IMAGE_ESRI_BETA_API_KEY: '${IMAGE_ESRI_BETA_API_KEY}'
+            IMAGE_MAPBOX_API_KEY: '${IMAGE_MAPBOX_API_KEY}'
+            IMAGE_MAXAR_PREMIUM_API_KEY: '${IMAGE_MAXAR_PREMIUM_API_KEY}'
+            IMAGE_MAXAR_STANDARD_API_KEY: '${IMAGE_MAXAR_STANDARD_API_KEY}'
+            SLACK_TOKEN: '${SLACK_TOKEN}'
+            SLACK_CHANNEL: '${SLACK_CHANNEL}'
+            SENTRY_DSN: '${SENTRY_DSN}'
         depends_on:
             - postgres
         command: mapswipe_workers --verbose run --schedule
@@ -67,8 +87,8 @@ services:
             context: postgres/
         environment:
             POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}'
-            POSTGRES_USER: 'mapswipe_workers'
-            POSTGRES_DB: 'mapswipe'
+            POSTGRES_USER: '${POSTGRES_USER}'
+            POSTGRES_DB: '${POSTGRES_DB}'
             PGDATA: '/var/lib/postgresql/mapswipe'
             WALG_GS_PREFIX: '${WALG_GS_PREFIX}'
             GOOGLE_APPLICATION_CREDENTIALS: '/serviceAccountKey.json'
@@ -88,8 +108,8 @@ services:
             context: postgres/recovery/
         environment:
             POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}'
-            POSTGRES_USER: 'mapswipe_workers'
-            POSTGRES_DB: 'mapswipe'
+            POSTGRES_USER: '${POSTGRES_USER}'
+            POSTGRES_DB: '${POSTGRES_DB}'
             PGDATA: '/var/lib/postgresql/mapswipe'
             WALG_GS_PREFIX: '${WALG_GS_PREFIX}'
             GOOGLE_APPLICATION_CREDENTIALS: '/serviceAccountKey.json'
@@ -104,6 +124,7 @@ services:
             context: firebase/
         environment:
             FIREBASE_TOKEN: '${FIREBASE_TOKEN}'
-        command: firebase deploy --token $FIREBASE_TOKEN --only functions,database:rules
+            FIREBASE_DB: '${FIREBASE_DB}'
+        command: sh -c "firebase use $FIREBASE_DB && firebase deploy --token $FIREBASE_TOKEN --only functions,database:rules"
         volumes:
             - ./firebase:/firebase

docker-compose.yaml

@@ -1,7 +1,9 @@
 # Configuration Reference
 
 Most of the configuration is stored in environment variables.
-At the root of the GitHub repository an example file (`example.env`) with all possible configuration variables exists. To get started copy this file to `.env` and fill in missing variables. Once done source this file to make variables accessible as environment variables: `source .env` to either be used by docker-compose during deployment setup or by MapSwipe Workers directly.
+At the root of the GitHub repository (in the same directory as `docker-compose.yml`) an example file (`example.env`) with all possible configuration variables exists. To get started copy this file to `.env` (no name is required) and fill in missing variables. The Docker Compose file will access those variables when needed.
+
+> Note: If you want those variables to be accessible as Environment Variables in your current shell (Eg. Inside a Python virtual environment for development.) your need to parse the file and export the variables: `export $(cat .env | xargs)`
 
 In following chapters configuration values and keys are discussed for each part of the MapSwipe Back-end.
 
@@ -13,14 +15,13 @@ All configuration values for MapSwipe Workers are stored in environment variable
 Required environment variables are:
 - FIREBASE_API_KEY
 - FIREBASE_DB
-- GOOGLE_APPLICATION_CREDENTIALS
 - POSTGRES_DB
 - POSTGRES_HOST
 - POSTGRES_PASSWORD
 - POSTGRES_PORT
 - POSTGRES_USER
 
-Mandatory environment variables are:
+Optional environment variables are:
 - SLACK_CHANNEL
 - SLACK_TOKEN
 - SENTRY_DSN
@@ -37,31 +38,47 @@ In addition to get access to Firebase a Service Account Key is required.
 The path the Service Account Key is defined in:
 - GOOGLE_APPLICATION_CREDENTIALS
 
-> Notes: When deploying using `docker` or `docker-compose` `POSTGRES_HOST` should have the value `postgres` and the Service Account Key (`serviceAccountKey.json`) should be copied to `mapswipe_workers/serviceAccountKey.json` as described in detail in [Deployment](deployment.md).
+> Notes: When deploying using `docker` or `docker-compose` `POSTGRES_HOST` should have the value `postgres` and the Service Account Key (`serviceAccountKey.json`) should be copied to `mapswipe_workers/serviceAccountKey.json` so that during the build of the image the file can by copied by Docker.
+
+
+### Elaboration
+
+**Firebase**: MapSwipe Workers use the Firebase Python SDK and the Firebase REST API. Both require the database name (`FIREBASE_DB`) and the API-Key from the Firebase instance. The Firebase Python SDK does also need a Service Account Key. The path to this file is set in the `GOOGLE_APPLICATION_CREDENTIALS` environment variable.
+
+**Postgres**: MapSwipe Workers writes data to a Postgres database and generate files for the API based data in Postgres.
+
+**Sentry (optional)**: MapSwipe workers use sentry to capture exceptions. You can find your project’s DSN in the “Client Keys” section of your “Project Settings” in Sentry. Check [Sentry's documentation](https://docs.sentry.io/error-reporting/configuration/?platform=python) for more information.
+
+**Slack (optional)**: The MapSwipe workers send messages to slack when a project has been created successfully, the project creation failed or an exception gets raised. refer to [Python slackclient's documentation](https://github.com/slackapi/python-slackclient) how to get a Slack Token.
+
+**Imagery:** MapSwipe uses satellite imagery provided by Tile Map Services (TMS).
+If you are not familiar with the basic concept have a look at [Bing's documentation](https://docs.microsoft.com/en-us/bingmaps/articles/bing-maps-tile-system).
 
 
 ## Postgres
 
-Required environment variables are:
+Required environment variables are (Those are the same as needed by MapSwipe Workers):
 - POSTGRES_DB
 - POSTGRES_HOST
 - POSTGRES_PASSWORD
 - POSTGRES_PORT
 - POSTGRES_USER
 
+> Notes: When deploying using `docker` or `docker-compose` `POSTGRES_HOST` should have the value `postgres`.
+
 
 ### Postgres Backup
 
-On details of how the back-up works please refer to [Postgres Backup](backup.md).
+On details of how the back-up works please refer to [Postgres Backup](backup.html).
 
 Required environment variables are:
 - WALG_GS_PREFIX
 
-To gain access to Google Cloud Storage another Service Account Key is needed. Again refer to [Postgres Backup](backup.md) on how to create this file.
+To gain access to Google Cloud Storage another Service Account Key is needed. Again refer to [Postgres Backup](backup.html) on how to create this file.
 The Service Account Key (`serviceAccountKey.json`) should be saved to `postgres/serviceAccountKey.json`
 
 
-### Manager Dashboard
+## Manager Dashboard
 
 `manager_dashboard/manager_dashboard/js/app.js`
 

docs/source/_static/img/area-plot.png

@@ -0,0 +1,70 @@
+# Data
+Swiping is just the beginning -- MapSwipe data is created by our users and accessible to the entire community. Through the [MapSwipe website](https://mapswipe.org/data.html) you can see where we've mapped, which organizations are requesting data, and how many individuals contribute to our impact. When using MapSwipe data, all you have to do is credit the MapSwipe contributors. Here you find a more detailed description of the data available.
+
+## Projects
+
+Files:
+- [projects.csv](https://apps.mapswipe.org/api/projects/projects.csv)
+- [projects_geom.geojson](https://apps.mapswipe.org/api/projects/projects_geom.geojson)
+- [projects_centroid.geojson](https://apps.mapswipe.org/api/projects/projects_centroid.geojson)
+
+Attributes:
+
+| Name | Type | Description |
+| ---- | ---- | ----------- |
+| idx  | integer | - |
+| project_id | string | The ID of the project used in Firebase. |
+| name | string | The name of the project as displayed in the app. For newer projects this will be a composition of project topic, project region and project number. |
+| project_details | string | The project description. This is displayed in the app on the project page. |
+| look_for | string | What should the users look for (e.g. buildings, cars, trees)? |
+| project_type | integer | The type of the project. 1=BuildArea, 2=Footprint, 3=ChangeDetection, 4=Completeness |
+| tile_server_names | string | The names of the satellite imagery providers used for this project. Some projects, e.g. Change Detection Type projects, have multiple tile server names. |
+| status | string | The status of the project defines if the project is visible in the app. There are the following values: active, inactive, finished, archived. Archived projects can not be set to active again. |
+| area_sqkm | float | The size of the project area in square kilometers. |
+| geom | string | The geometry of the project region as WKT geometry. |
+| centroid | string | The centroid of the project geometry as WKT geometry. |
+| progress | float | The mapping progress of the project. 1.0=completed. |
+| number_of_users | integer | The number of distinct users who contributed to this project. |
+| number_of_results | integer | The total number of results for all tasks. |
+| number_of_results_progress | integer | The number of results that are taken into account for the progress calculation. If a tasks has been mapped more often than required, usually 3-times, the additional results are not considered for the progress calculation. |
+| day | string | The date when the project information was updated the last time. |
+
+
+## Aggregated Results
+This gives you the unfiltered MapSwipe results. This is most suited if you want to apply some custom data processing with the MapSwipe data, e.g. select only specific tasks for machine learning. If you want to use MapSwipe data in the Tasking Manager you might look for the data explained below.
+
+Files:
+- `aggregated_results_{project_id}.csv`, e.g. [agg\_results\_-M56eeMCZ5VeOHjJN4Bx.csv](https://apps.mapswipe.org/api/agg_results/agg_results_-M56eeMCZ5VeOHjJN4Bx.csv)
+- `aggregated_results_{project_id}.geojson`, e.g. [agg\_results\_-M56eeMCZ5VeOHjJN4Bx.geojson](https://apps.mapswipe.org/api/agg_results/agg_results_-M56eeMCZ5VeOHjJN4Bx.geojson)
+
+| Name | Type | Description |
+| ---- | ---- | ----------- |
+| idx  | integer | - |
+| task_id | string | The ID of the task, for BuildArea projects this is a composition of `TileZ-TileX-TileY` |
+| 0_count | integer | The number of users who marked this task as 0, e.g. "no building" for BuildArea Project Type. |
+| 1_count | integer | The number of users who marked this task as 1, e.g. "building" for BuildArea Project Type. |
+| 2_count | integer | The number of users who marked this task as 2, e.g. "maybe" for BuildArea Project Type. |
+| 3_count | integer | The number of users who marked this task as 3, e.g. "bad imagery" for BuildArea Project Type. |
+| total_count | integer | The total number of users who mapped this task. |
+| 0_share | float | 0_count divived by total_count. This gives you the share of all users who marked as 0. |
+| 1_share | float | 1_count divived by total_count. This gives you the share of all users who marked as 1. |
+| 2_share | float | 2_count divived by total_count. This gives you the share of all users who marked as 2. |
+| 3_share | float | 3_count divived by total_count. This gives you the share of all users who marked as 3. |
+| agreement | float | This is defined as [Scott's Pi](https://en.wikipedia.org/wiki/Scott%27s_Pi) and gives you an understanding of inter-rater reliability. The value is 1.0 if all users agree, e.g. all users classify as "building". If users disagree this value will be lower. |
+ | geom | string | The geometry of this task as WKT geometry. |
+ 
+ 
+ ## HOT Tasking Manager Geometries
+ This gives you filtered MapSwipe data ready to be imported to the HOT Tasking Manager. Currently, the geometries in this dataset consist of maximum 15 MapSwipe Tasks, where at least 35% of all users indicated the presence of a building by classifying as "yes" or "maybe". 
+ 
+ Files:
+ - `hot_tm_{project_id}.geojson`, e.g. [hot\_tm\_-M56eeMCZ5VeOHjJN4Bx.geojson](https://apps.mapswipe.org/api/hot_tm/hot_tm_-M56eeMCZ5VeOHjJN4Bx.geojson)
+ 
+| Name | Type | Description |
+| ---- | ---- | ----------- |
+| group_id | integer | A ID for the geometry. It has no connection to the MapSwipe data model. |
+| geometry | geometry | A polygon geometry representing the selected MapSwipe tasks. In our GIS workflow we further aggregate and simplify the geometry, hence they kind of look like easter eggs. |
+
+<img src="_static/img/tasking_manager_geometries_example.png" width="600px">
+ 
+ 

docs/source/_static/img/chad_area.png

@@ -3,11 +3,13 @@
 ## Logs - MapSwipe Workers
 
 Where can I find logs?
+- `docker logs mapswipe_workers`
+- or
+- `cat mapswipe-data/mapswipe_workers.log`
 
-Logs are written to directly to console and also to `/var/log/mapswipe_workers.log`:
+Logs are written to directly to the terminal (stdout). The easiest way is therefore is to run `docker logs mapswipe_workers` to see the logs.
 
-- `docker exec -t mapswipe_workers cat /var/log/mapswipe_workers/mapswipe_workers.log` (if container is running)
-- `docker logs container_name` (also if container is not running)
+Logs are also writing to file inside the Docker container (`~/.local/share/mapswipe_workers/mapswipe_workers.log`). The parent directory of the file is the data directory of MapSwipe Workers. This directory is mounted (as a Docker volume) locally to disk (`mapswipe-data/`). Logs can therefore be accessed as text file as well: `cat mapswipe-data/mapswipe_workers.log`
 
 
 ## Common Errors