Better data transfer firebase postgres (#464)

* improve update project data function by using threading for firebase queries and check project status before updating
* work on faster workflow to update user data in postgres #462
* split up workflow for generate stats and firebase-to-postgres into two docker containers
* small fix in docker compose file
* adjust archive function to remove also new firebase endpoint for /v2/groupsUsers and adjust tests. this also relates to #416
* add script to remove timestamps in user contributions. this also relates to #375 #376
* consider new firebae endpoint /v2/groupsUsers/ when deleting projects
* update user data only when new users submit a result.

Author: Hagellach37

docker-compose.yaml

@@ -75,7 +75,7 @@ services:
             SENTRY_DSN: '${SENTRY_DSN}'
         depends_on:
             - postgres
-        command: mapswipe_workers --verbose run --analysis_type=creation --schedule
+        command: mapswipe_workers --verbose run --analysis_type=creation --schedule --time_interval=5
         volumes:
             - ./mapswipe-data:/root/.local/share/mapswipe_workers
         restart: "no"
@@ -109,7 +109,41 @@ services:
             SENTRY_DSN: '${SENTRY_DSN}'
         depends_on:
             - postgres
-        command: mapswipe_workers --verbose run --analysis_type=stats --schedule
+        command: mapswipe_workers --verbose run --analysis_type=generate-stats --schedule --time_interval=60
+        volumes:
+            - ./mapswipe-data:/root/.local/share/mapswipe_workers
+        restart: "no"
+        networks:
+            - mapswipe_workers
+            - postgres
+
+    mapswipe_workers_firebase_to_postgres:
+        container_name: mapswipe_workers_firebase_to_postgres
+        build:
+            context: mapswipe_workers/
+        environment:
+            FIREBASE_DB: '${FIREBASE_DB}'
+            FIREBASE_API_KEY: '${FIREBASE_API_KEY}'
+            FIREBASE_TOKEN: '${FIREBASE_TOKEN}'
+            GOOGLE_APPLICATION_CREDENTIALS: 'serviceAccountKey.json'
+            POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}'
+            POSTGRES_USER: '${POSTGRES_USER}'
+            POSTGRES_DB: '${POSTGRES_DB}'
+            POSTGRES_HOST: 'postgres'
+            POSTGRES_PORT: 5432
+            PGDATA: '/var/lib/postgresql/mapswipe'
+            IMAGE_BING_API_KEY: '${IMAGE_BING_API_KEY}'
+            IMAGE_ESRI_API_KEY: '${IMAGE_ESRI_API_KEY}'
+            IMAGE_ESRI_BETA_API_KEY: '${IMAGE_ESRI_BETA_API_KEY}'
+            IMAGE_MAPBOX_API_KEY: '${IMAGE_MAPBOX_API_KEY}'
+            IMAGE_MAXAR_PREMIUM_API_KEY: '${IMAGE_MAXAR_PREMIUM_API_KEY}'
+            IMAGE_MAXAR_STANDARD_API_KEY: '${IMAGE_MAXAR_STANDARD_API_KEY}'
+            SLACK_TOKEN: '${SLACK_TOKEN}'
+            SLACK_CHANNEL: '${SLACK_CHANNEL}'
+            SENTRY_DSN: '${SENTRY_DSN}'
+        depends_on:
+            - postgres
+        command: mapswipe_workers --verbose run --analysis_type=firebase-to-postgres --schedule --time_interval=2
         volumes:
             - ./mapswipe-data:/root/.local/share/mapswipe_workers
         restart: "no"

mapswipe_workers/Dockerfile

@@ -1,5 +1,5 @@
 # https://github.com/OSGeo/gdal/tree/master/gdal/docker
-# Image includes python3.6, gdal-python, gdal-bin
+# Image includes python3.8, gdal-python, gdal-bin
 FROM osgeo/gdal:ubuntu-small-latest
 
 # Set C.UTF-8 locale as default (Needed by the Click library)

mapswipe_workers/mapswipe_workers/firebase_to_postgres/archive_project.py

@@ -2,6 +2,7 @@
 Archive a project.
 """
 import re
+import time
 from typing import Iterable
 
 from firebase_admin import exceptions
@@ -59,11 +60,21 @@ def archive_project(project_ids: list) -> bool:
                 ref.update({key: None for key in chunk})
             ref.delete()
 
+        ref = fb_db.reference(f"v2/groupsUsers/{project_id}")
+        if not re.match(r"/v2/\w+/[-a-zA-Z0-9]+", ref.path):
+            raise CustomError(
+                f"""Given argument resulted in invalid Firebase Realtime Database reference.
+                        {ref.path}"""
+            )
+        ref.delete()
+
+        time.sleep(5)  # Wait for Firebase Functions to complete
+
         ref = fb_db.reference(f"v2/groups/{project_id}")
         if not re.match(r"/v2/\w+/[-a-zA-Z0-9]+", ref.path):
             raise CustomError(
                 f"""Given argument resulted in invalid Firebase Realtime Database reference.
-                {ref.path}"""
+                        {ref.path}"""
             )
         ref.delete()
 

mapswipe_workers/mapswipe_workers/firebase_to_postgres/delete_project.py

@@ -2,6 +2,7 @@
 Delete projects.
 """
 import re
+import time
 from typing import Iterable
 
 from firebase_admin import exceptions
@@ -58,6 +59,15 @@ def delete_project(project_ids: list) -> bool:
                 ref.update({key: None for key in chunk})
             ref.delete()
 
+        ref = fb_db.reference(f"v2/groupsUsers/{project_id}")
+        if not re.match(r"/v2/\w+/[-a-zA-Z0-9]+", ref.path):
+            raise CustomError(
+                f"""Given argument resulted in invalid Firebase Realtime Database reference.
+                                {ref.path}"""
+            )
+        ref.delete()
+        time.sleep(5)  # Wait for Firebase Functions to complete
+
         ref = fb_db.reference(f"v2/groups/{project_id}")
         if not re.match(r"/v2/\w+/[-a-zA-Z0-9]+", ref.path):
             raise CustomError(