Use GitHub actions instead of Travis (#469)

Author: Hagellach37

.github/workflows/actions.yml

@@ -0,0 +1,64 @@
+name: MapSwipe Workers
+
+on: [push]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.8]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      working-directory: ./mapswipe_workers
+      run: |
+        python -m pip install --upgrade pip
+        pip install flake8 black==19.10b0 isort
+    - name: Code style
+      working-directory: ./mapswipe_workers
+      run: |
+        black --check mapswipe_workers
+        flake8 --count --config setup.cfg mapswipe_workers/
+        isort --check --settings-file setup.cfg mapswipe_workers/
+    - name:  Setup Postgres Database Container
+      env:
+        POSTGRES_PASSWORD: test
+        POSTGRES_USER: test
+        POSTGRES_DB: test
+      run: |
+        # Create a mock file for wal-g setup
+        touch postgres/serviceAccountKey.json
+        docker-compose up --build --detach postgres
+    - name: Deploy Firebase Rules and Functions
+      env:
+        FIREBASE_TOKEN: ${{ secrets.FIREBASE_TOKEN }}
+        FIREBASE_DB: ${{ secrets.FIREBASE_DB }}
+      run: |
+        # Create a mock file for wal-g setup
+        docker-compose up --build firebase_deploy
+    - name: Decrypt Service Account Key File
+      working-directory: ./
+      run: |
+        openssl enc -aes-256-cbc -d -K "$OPENSSL_KEY" -iv "$OPENSSL_IV" -in ci-mapswipe-firebase-adminsdk-80fzw-ebce84bd5b.json.enc -out mapswipe_workers/serviceAccountKey.json
+      env:
+        OPENSSL_PASSPHRASE: ${{ secrets.OPENSSL_PASSPHRASE }}
+        OPENSSL_KEY: ${{ secrets.OPENSSL_KEY }}
+        OPENSSL_IV: ${{ secrets.OPENSSL_IV }}
+    - name: Run Tests
+      working-directory: ./mapswipe_workers
+      env:
+        FIREBASE_API_KEY: ${{ secrets.FIREBASE_API_KEY }}
+        FIREBASE_DB: ${{ secrets.FIREBASE_DB }}
+        POSTGRES_PASSWORD: test
+        POSTGRES_USER: test
+        POSTGRES_DB: test
+      run: |
+        docker-compose run mapswipe_workers_creation python -m unittest discover --verbose --start-directory tests/unittests/
+        docker-compose run mapswipe_workers_creation python -m unittest discover --verbose --start-directory tests/integration/

.travis.yml

@@ -25,3 +25,9 @@ This is done by using Travis script deployment (https://docs.travis-ci.com/user/
 To be able to connect to the MapSwipe server the Travis instance uses an encrypted SSH private key (Which can be found in the directory `travis/`).
 
 In the `deploy.sh` script an Ansible Playbook is run (https://docs.ansible.com/ansible/latest/index.html). Ansible is an automation tool which utilizes a SSH connection (`ansible/ansible.cfg`) to run commands defined in the Playbook (`ansible/playbook.yml`) on hosts defined in the Inventory (`ansible/inventory.yml`).
+
+## Continuous Deployment with Github Actions
+We use an encrypted service account key file. The file has been generated with this commands:
+
+`openssl enc -aes-256-cbc -e -p -nosalt -in=ci-mapswipe-firebase-adminsdk-80fzw-ebce84bd5b.json -out=ci-mapswipe-firebase-adminsdk-80fzw-ebce84bd5b.json.enc`
+

ci-mapswipe-firebase-adminsdk-80fzw-ebce84bd5b.json.enc

@@ -133,15 +133,16 @@ def set_progress_in_firebase(project_id: str):
     query = """
         -- Calculate overall project progress as
         -- the average progress for all groups.
-        -- This is not hundred percent exact, since groups can have a different number of tasks
+        -- This is not hundred percent exact
+        -- since groups can have a different number of tasks
         -- but it is still "good enough" and gives almost correct progress.
         -- But it is easier to compute
         -- than considering the actual number of tasks per group.
         select
           project_id
           ,avg(group_progress)::integer as progress
         from
-        (   
+        (
             -- Get all groups for this project and
             -- add progress for groups that have been worked on already.
             -- Set progress to 0 if no user has worked on this group.
@@ -155,7 +156,7 @@ def set_progress_in_firebase(project_id: str):
                 else group_progress
               end as group_progress
             from groups g
-            left join 
+            left join
                 (
                 -- Here we get the progress for all groups
                 -- for which results have been submitted already.
@@ -170,13 +171,13 @@ def set_progress_in_firebase(project_id: str):
                     else 100 * count(distinct user_id) / p.verification_number
                   end as group_progress
                 from results r, projects p
-                where r.project_id = p.project_id 
-                group by group_id, r.project_id, p.verification_number 
+                where r.project_id = p.project_id
+                group by group_id, r.project_id, p.verification_number
             ) bar
             on bar.group_id = g.group_id and bar.project_id = g.project_id
-            where g.project_id = %s 
-        ) foo	
-        group by project_id 
+            where g.project_id = %s
+        ) foo
+        group by project_id
     """
     data = [project_id]
     progress = pg_db.retr_query(query, data)[0][1]
@@ -195,15 +196,17 @@ def set_contributor_count_in_firebase(project_id: str):
         select
           project_id
           ,count(distinct user_id) contributor_count
-        from results r 
-        where 
-          project_id = %s 
-        group by project_id 
+        from results r
+        where
+          project_id = %s
+        group by project_id
     """
     data = [project_id]
     contributor_count = pg_db.retr_query(query, data)[0][1]
 
     fb_db = auth.firebaseDB()
     project_progress_ref = fb_db.reference(f"v2/projects/{project_id}/contributorCount")
     project_progress_ref.set(contributor_count)
-    logger.info(f"set contributorCount attribute for project {project_id}: {contributor_count}")
+    logger.info(
+        f"set contributorCount attribute for project {project_id}: {contributor_count}"
+    )

docs/source/deployment_overview.md

@@ -1,8 +1,8 @@
 import datetime
-import os
-from typing import List
 import gzip
+import os
 import tempfile
+from typing import List
 
 import pandas as pd
 from psycopg2 import sql
@@ -34,12 +34,12 @@ def write_sql_to_gzipped_csv(filename: str, sql_query: sql.SQL):
     """
 
     # generate temporary file which will be automatically deleted at the end
-    tmp_csv_file = os.path.join(tempfile._get_default_tempdir(), 'tmp.csv')
+    tmp_csv_file = os.path.join(tempfile._get_default_tempdir(), "tmp.csv")
     pg_db = auth.postgresDB()
     with open(tmp_csv_file, "w") as f:
         pg_db.copy_expert(sql_query, f)
 
-    with open(tmp_csv_file, 'rb') as f_in, gzip.open(filename, 'wb') as f_out:
+    with open(tmp_csv_file, "rb") as f_in, gzip.open(filename, "wb") as f_out:
         f_out.writelines(f_in)
 
     logger.info(f"wrote gzipped csv file from sql: {filename}")
@@ -52,11 +52,7 @@ def load_df_from_csv(filename: str) -> pd.DataFrame:
     """
     dtype_dict = {"project_id": str, "group_id": str, "task_id": str}
 
-    df = pd.read_csv(
-        filename,
-        dtype=dtype_dict,
-        compression="gzip"
-    )
+    df = pd.read_csv(filename, dtype=dtype_dict, compression="gzip")
     logger.info(f"loaded pandas df from {filename}")
     return df
 
@@ -337,7 +333,9 @@ def get_per_project_statistics(project_id: str, project_info: pd.Series) -> dict
     results_filename = f"{DATA_PATH}/api/results/results_{project_id}.csv.gz"
     tasks_filename = f"{DATA_PATH}/api/tasks/tasks_{project_id}.csv.gz"
     groups_filename = f"{DATA_PATH}/api/groups/groups_{project_id}.csv.gz"
-    agg_results_filename = f"{DATA_PATH}/api/agg_results/agg_results_{project_id}.csv.gz"
+    agg_results_filename = (
+        f"{DATA_PATH}/api/agg_results/agg_results_{project_id}.csv.gz"
+    )
     agg_results_by_user_id_filename = f"{DATA_PATH}/api/users/users_{project_id}.csv.gz"
     project_stats_by_date_filename = f"{DATA_PATH}/api/history/history_{project_id}.csv"
 
@@ -358,15 +356,12 @@ def get_per_project_statistics(project_id: str, project_info: pd.Series) -> dict
 
         # aggregate results by task id
         agg_results_df = get_agg_results_by_task_id(results_df, tasks_df)
-        agg_results_df.to_csv(
-            agg_results_filename,
-            index_label="idx"
-        )
+        agg_results_df.to_csv(agg_results_filename, index_label="idx")
 
         geojson_functions.gzipped_csv_to_gzipped_geojson(
             filename=agg_results_filename,
             geometry_field="geom",
-            add_metadata=add_metadata
+            add_metadata=add_metadata,
         )
         logger.info(f"saved agg results for {project_id}: {agg_results_filename}")
 
@@ -377,8 +372,7 @@ def get_per_project_statistics(project_id: str, project_info: pd.Series) -> dict
                 results_df, agg_results_df
             )
             agg_results_by_user_id_df.to_csv(
-                agg_results_by_user_id_filename,
-                index_label="idx"
+                agg_results_by_user_id_filename, index_label="idx"
             )
             logger.info(
                 f"saved agg results for {project_id}: {agg_results_by_user_id_filename}"
@@ -404,8 +398,7 @@ def get_per_project_statistics(project_id: str, project_info: pd.Series) -> dict
             logger.info(f"do NOT generate tasking manager geometries for {project_id}")
         else:
             tasking_manager_geometries.generate_tasking_manager_geometries(
-                project_id=project_id,
-                agg_results_filename=agg_results_filename
+                project_id=project_id, agg_results_filename=agg_results_filename
             )
 
         # prepare output of function

mapswipe_workers/mapswipe_workers/firebase_to_postgres/update_data.py

@@ -64,7 +64,8 @@ def get_progress_by_date(
         - sum of daily number of results up to that day
 
     progress:
-        - relative progress per day (e.g. overall progress increased by 0.15 on that day)
+        - relative progress per day
+        (e.g. overall progress increased by 0.15 on that day)
         - not used in firebase
 
     cum_progress:

mapswipe_workers/mapswipe_workers/generate_stats/project_stats.py

@@ -100,9 +100,7 @@ def run_create_projects():
             except UnboundLocalError:
                 project_id = None
 
-            send_slack_message(
-                MessageType.FAIL, project_name, project_id, str(e)
-            )
+            send_slack_message(MessageType.FAIL, project_name, project_id, str(e))
             logger.exception("Failed: Project Creation ({0}))".format(project_name))
             sentry.capture_exception()
         continue
@@ -386,15 +384,15 @@ def run_delete_project(project_id, project_ids):
 
 @cli.command("run")
 @click.option(
-        "--analysis_type",
-        "-a",
-        default="all",
-        help=(
-            f"Will either execute all or just the specified functions of the analysis "
-            f"choices here"
-        ),
-        type=click.Choice(["all", "creation", "stats"]),
-    )
+    "--analysis_type",
+    "-a",
+    default="all",
+    help=(
+        "Will either execute all or just the specified functions of the analysis "
+        "choices here"
+    ),
+    type=click.Choice(["all", "creation", "stats"]),
+)
 @click.option("--schedule", is_flag=True, help="Schedule jobs to run every 10 minutes.")
 @click.pass_context
 def run(context, analysis_type, schedule):

mapswipe_workers/mapswipe_workers/generate_stats/project_stats_by_date.py

@@ -51,4 +51,3 @@ def check_imagery_url(self):
             return True
         else:
             return False
-

mapswipe_workers/mapswipe_workers/mapswipe_workers.py

@@ -1,6 +1,6 @@
+import gzip
 import json
 import os
-import gzip
 import shutil
 import subprocess
 import tempfile
@@ -11,9 +11,7 @@
 
 
 def gzipped_csv_to_gzipped_geojson(
-        filename: str,
-        geometry_field: str = "geom",
-        add_metadata: bool = False
+    filename: str, geometry_field: str = "geom", add_metadata: bool = False
 ):
     """Convert gzipped csv file to gzipped GeoJSON.
 
@@ -22,13 +20,13 @@ def gzipped_csv_to_gzipped_geojson(
     Last, the generated geojson file is again compressed using gzip.
     """
     # generate temporary files which will be automatically deleted at the end
-    tmp_csv_file = os.path.join(tempfile._get_default_tempdir(), 'tmp.csv')
-    tmp_geojson_file = os.path.join(tempfile._get_default_tempdir(), 'tmp.geojson')
+    tmp_csv_file = os.path.join(tempfile._get_default_tempdir(), "tmp.csv")
+    tmp_geojson_file = os.path.join(tempfile._get_default_tempdir(), "tmp.geojson")
 
     outfile = filename.replace(".csv", f"_{geometry_field}.geojson")
 
     # uncompress content of zipped csv file and save to csv file
-    with gzip.open(filename, 'rb') as f_in:
+    with gzip.open(filename, "rb") as f_in:
         with open(tmp_csv_file, "wb") as f_out:
             shutil.copyfileobj(f_in, f_out)
 
@@ -56,7 +54,7 @@ def gzipped_csv_to_gzipped_geojson(
     with open(tmp_geojson_file, "r") as f:
         json_data = json.load(f)
 
-    with gzip.open(outfile, 'wt') as fout:
+    with gzip.open(outfile, "wt") as fout:
         json.dump(json_data, fout)
 
     logger.info(f"converted {filename} to {outfile} with ogr2ogr.")