Merge pull request #458 from mapswipe/separate-docker-containers

split workers into two separate docker containers

Author: Hagellach37

.travis.yml

@@ -35,8 +35,8 @@ script:
   - isort --check --settings-file mapswipe_workers/setup.cfg mapswipe_workers
   - docker-compose up --build --detach firebase_deploy
   - docker-compose up --build --detach postgres
-  - docker-compose run mapswipe_workers python -m unittest discover --verbose --start-directory tests/unittests/
-  - docker-compose run mapswipe_workers python -m unittest discover --verbose --start-directory tests/integration/
+  - docker-compose run mapswipe_workers_creation python -m unittest discover --verbose --start-directory tests/unittests/
+  - docker-compose run mapswipe_workers_creation python -m unittest discover --verbose --start-directory tests/integration/
 
 after_success:
   # SSH setup to deploy to server after build.

ansible/playbook.yml

@@ -9,10 +9,10 @@
         dest: $HOME/python-mapswipe-workers
         version: dev
     - name: Rebuild Docker container.
-      command: docker-compose build postgres firebase_deploy mapswipe_workers manager_dashboard nginx api
+      command: docker-compose build postgres firebase_deploy mapswipe_workers_creation mapswipe_workers_stats manager_dashboard nginx api
       args:
         chdir: $HOME/python-mapswipe-workers
     - name: Restart Docker container.
-      command: docker-compose up -d --force-recreate postgres firebase_deploy mapswipe_workers manager_dashboard nginx api
+      command: docker-compose up -d --force-recreate postgres firebase_deploy mapswipe_workers_creation mapswipe_workers_stats manager_dashboard nginx api
       args:
         chdir: $HOME/python-mapswipe-workers

docker-compose.yaml

@@ -49,7 +49,41 @@ services:
         networks:
             - manager_dashboard
 
-    mapswipe_workers:
+    mapswipe_workers_creation:
+        container_name: mapswipe_workers
+        build:
+            context: mapswipe_workers/
+        environment:
+            FIREBASE_DB: '${FIREBASE_DB}'
+            FIREBASE_API_KEY: '${FIREBASE_API_KEY}'
+            FIREBASE_TOKEN: '${FIREBASE_TOKEN}'
+            GOOGLE_APPLICATION_CREDENTIALS: 'serviceAccountKey.json'
+            POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}'
+            POSTGRES_USER: '${POSTGRES_USER}'
+            POSTGRES_DB: '${POSTGRES_DB}'
+            POSTGRES_HOST: 'postgres'
+            POSTGRES_PORT: 5432
+            PGDATA: '/var/lib/postgresql/mapswipe'
+            IMAGE_BING_API_KEY: '${IMAGE_BING_API_KEY}'
+            IMAGE_ESRI_API_KEY: '${IMAGE_ESRI_API_KEY}'
+            IMAGE_ESRI_BETA_API_KEY: '${IMAGE_ESRI_BETA_API_KEY}'
+            IMAGE_MAPBOX_API_KEY: '${IMAGE_MAPBOX_API_KEY}'
+            IMAGE_MAXAR_PREMIUM_API_KEY: '${IMAGE_MAXAR_PREMIUM_API_KEY}'
+            IMAGE_MAXAR_STANDARD_API_KEY: '${IMAGE_MAXAR_STANDARD_API_KEY}'
+            SLACK_TOKEN: '${SLACK_TOKEN}'
+            SLACK_CHANNEL: '${SLACK_CHANNEL}'
+            SENTRY_DSN: '${SENTRY_DSN}'
+        depends_on:
+            - postgres
+        command: mapswipe_workers --verbose run --analysis_type=creation --schedule
+        volumes:
+            - ./mapswipe-data:/root/.local/share/mapswipe_workers
+        restart: "no"
+        networks:
+            - mapswipe_workers
+            - postgres
+
+    mapswipe_workers_stats:
         container_name: mapswipe_workers
         build:
             context: mapswipe_workers/
@@ -75,7 +109,7 @@ services:
             SENTRY_DSN: '${SENTRY_DSN}'
         depends_on:
             - postgres
-        command: mapswipe_workers --verbose run --schedule
+        command: mapswipe_workers --verbose run --analysis_type=stats --schedule
         volumes:
             - ./mapswipe-data:/root/.local/share/mapswipe_workers
         restart: "no"

mapswipe_workers/mapswipe_workers/mapswipe_workers.py

@@ -383,9 +383,19 @@ def run_delete_project(project_id, project_ids):
 
 
 @cli.command("run")
+@click.option(
+        "--analysis_type",
+        "-a",
+        default="all",
+        help=(
+            f"Will either execute all or just the specified functions of the analysis "
+            f"choices here"
+        ),
+        type=click.Choice(["all", "creation", "stats"]),
+    )
 @click.option("--schedule", is_flag=True, help="Schedule jobs to run every 10 minutes.")
 @click.pass_context
-def run(context, schedule):
+def run(context, analysis_type, schedule):
     """
     Run all commands.
 
@@ -400,10 +410,30 @@ def _run():
         project_ids = context.invoke(run_firebase_to_postgres)
         context.invoke(run_generate_stats, project_ids=project_ids)
 
+    def _run_creation():
+        logger.info("start mapswipe backend workflow for creation.")
+        context.invoke(run_create_projects)
+        context.invoke(run_create_tutorials)
+
+    def _run_stats():
+        logger.info("start mapswipe backend workflow for stats.")
+        project_ids = context.invoke(run_firebase_to_postgres)
+        context.invoke(run_generate_stats, project_ids=project_ids)
+
     if schedule:
-        sched.every(10).minutes.do(_run).run()
+        if analysis_type == "all":
+            sched.every(10).minutes.do(_run).run()
+        elif analysis_type == "creation":
+            sched.every(10).minutes.do(_run_creation).run()
+        elif analysis_type == "stats":
+            sched.every(10).minutes.do(_run_stats).run()
         while True:
             sched.run_pending()
             time.sleep(1)
     else:
-        _run()
+        if analysis_type == "all":
+            sched.every(10).minutes.do(_run).run()
+        elif analysis_type == "creation":
+            sched.every(10).minutes.do(_run_creation).run()
+        elif analysis_type == "stats":
+            sched.every(10).minutes.do(_run_stats).run()

recreate_container.sh

@@ -2,15 +2,15 @@
 # This script builds and starts all Docker container for running the Mapswipe ecosystem.
 # It is run either manually or by an Ansible Playbook after a successful Travis build.
 
-docker-compose build --no-cache postgres firebase_deploy mapswipe_workers manager_dashboard nginx api
+docker-compose build --no-cache postgres firebase_deploy mapswipe_workers_creation mapswipe_workers_stats manager_dashboard nginx api
 if [[ $? = 0 ]]; then
     echo "success"
 else
     echo "failure: $?"
     exit
 fi
 
-docker-compose up -d --force-recreate postgres firebase_deploy mapswipe_workers manager_dashboard nginx api
+docker-compose up -d --force-recreate postgres firebase_deploy mapswipe_workers_creation mapswipe_workers_stats manager_dashboard nginx api
 if [[ $? = 0 ]]; then
     echo "success"
 else