Merge branch 'dev' into docs

Author: matthias_schaub

docker-compose.yaml

@@ -53,14 +53,16 @@ services:
             context: mapswipe_workers/
         depends_on:
             - postgres
-        command: mapswipe_workers --verbose run --schedule m
+        command: mapswipe_workers --verbose run --schedule m --only_new_results
         volumes:
             - ./api-data:/var/lib/mapswipe_workers/api-data/:rw
-            - ./api-data/agg_res_by_task_id:/var/lib/mapswipe_workers/api-data/agg_res_by_task_id:rw
-            - ./api-data/agg_res_by_project_id_and_date:/var/lib/mapswipe_workers/api-data/agg_res_by_project_id_and_date:rw
-            - ./api-data/agg_progress_by_project_id_and_date:/var/lib/mapswipe_workers/api-data/agg_progress_by_project_id_and_date:rw
-            - ./api-data/agg_res_by_user_id_and_date:/var/lib/mapswipe_workers/api-data/agg_res_by_user_id_and_date:rw
-        restart: unless-stopped
+            - ./api-data/agg_results:/var/lib/mapswipe_workers/api-data/agg_results:rw
+            - ./api-data/groups:/var/lib/mapswipe_workers/api-data/groups:rw
+            - ./api-data/history:/var/lib/mapswipe_workers/api-data/history:rw
+            - ./api-data/projects:/var/lib/mapswipe_workers/api-data/projects:rw
+            - ./api-data/results:/var/lib/mapswipe_workers/api-data/results:rw
+            - ./api-data/tasks:/var/lib/mapswipe_workers/api-data/tasks:rw
+        restart: "no"
         networks:
             - mapswipe_workers
             - postgres
@@ -70,38 +72,37 @@ services:
         build:
             context: postgres/
         environment:
-            POSTGERS_PASSWORD: '${POSTGRES_PASSWORD}'
+            POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}'
             POSTGRES_USER: 'mapswipe_workers'
             POSTGRES_DB: 'mapswipe'
+            PGDATA: '/var/lib/postgresql/mapswipe'
             WALG_GS_PREFIX: '${WALG_GS_PREFIX}'
             GOOGLE_APPLICATION_CREDENTIALS: '/serviceAccountKey.json'
         # Set options for WAL-G (backup tool)
         command: postgres -c archive_mode=on -c archive_timeout=60 -c archive_command="/archive_command.sh %p"
         volumes:
-            - ./postgres-data:/var/lib/postgresql/data
+          - ./postgres-data:/var/lib/postgresql/mapswipe
         restart: unless-stopped
         ports:
           - "5432:5432"
         networks:
             - postgres
 
-    postgres_restore_backup:
-        container_name: postgres_restore_backup
+    postgres_recovery:
+        container_name: postgres_recovery
         build:
-            context: postgres/
-            dockerfile: Dockerfile-restore_backup
-            args:
-                WALG_GS_PREFIX: '${WALG_GS_PREFIX}'
-                GOOGLE_APPLICATION_CREDENTIALS: '/serviceAccountKey.json'
+            context: postgres/recovery/
         environment:
             POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}'
             POSTGRES_USER: 'mapswipe_workers'
             POSTGRES_DB: 'mapswipe'
-        command: postgres
+            PGDATA: '/var/lib/postgresql/mapswipe'
+            WALG_GS_PREFIX: '${WALG_GS_PREFIX}'
+            GOOGLE_APPLICATION_CREDENTIALS: '/serviceAccountKey.json'
         volumes:
-            - ./postgres-data:/var/lib/postgresql/restored_cluster
+            - ./pgdata_restored:/var/lib/postgresql/mapswipe
         ports:
-          - "5432:5432"
+          - "5445:5432"
 
     firebase_deploy:
         container_name: firebase_deploy

docs/source/cli.md

@@ -4,6 +4,11 @@ This document describes how to use the command line interface of MapSwipe Worker
 
 In our current deployment setup the commands of the MapSwipe Workers CLI are hard-coded in the Docker-Compose File.
 
+You can run these commands also using docker-compose:
+```
+docker-compose run mapswipe_workers mapswipe_workers --help
+```
+
 
 ```
 Usage: mapswipe_workers [OPTIONS] COMMAND [ARGS]...
@@ -56,8 +61,26 @@ Usage: mapswipe_workers generate-stats [OPTIONS]
 Options:
   -s, --schedule [m|h|d]  Generate stats every 10 minutes (m), every hour (h)
                           or every day (d).
-  --only_new_results      Generate stats for all projects or only for those
-                          updated or with new results.
+  --project_id_list TEXT  provide project id strings as a list stats will be
+                          generated only for these projects.
+                          Use it like '["project_a", "project_b"]'
+  --help                  Show this message and exit.
+```
+
+## Generate Statistics for all projects
+
+Ideally you run this using a separate docker container. e.g. like this:
+
+```
+docker-compose run mapswipe_workers mapswipe_workers generate-stats-all-projects
+```
+
+```
+Usage: mapswipe_workers generate-stats-all-projects [OPTIONS]
+
+Options:
+  -s, --schedule [m|h|d]  Generate stats every 10 minutes (m), every hour (h)
+                          or every day (d).
   --help                  Show this message and exit.
 ```
 

firebase/functions/index.js

@@ -19,6 +19,27 @@ exports.counter = functions.database.ref('/v2/results/{projectId}/{groupId}/{use
     const promises = []
     const result = snapshot.after.val()
 
+    // if result ref does not contain all required attributes we don't updated counters
+    // e.g. due to some error when uploading from client
+    if (!result.hasOwnProperty('results')) {
+        console.log('no results attribute for /v2/results/'+context.params.projectId+'/'+context.params.groupId+'/'+context.params.userId )
+        console.log('will not update counters')
+        return null
+    } else if (!result.hasOwnProperty('timestamp')) {
+        console.log('no timestamp attribute for /v2/results/'+context.params.projectId+'/'+context.params.groupId+'/'+context.params.userId )
+        console.log('will not update counters')
+        return null
+    } else if (!result.hasOwnProperty('endTime')) {
+        console.log('no endTime attribute for /v2/results/'+context.params.projectId+'/'+context.params.groupId+'/'+context.params.userId )
+        console.log('will not update counters')
+        return null
+    } else if (!result.hasOwnProperty('startTime')) {
+        console.log('no startTime attribute for /v2/results/'+context.params.projectId+'/'+context.params.groupId+'/'+context.params.userId )
+        console.log('will not update counters')
+        return null
+    }
+
+
     // Firebase Realtime Database references
     const groupFinishedCountRef = admin.database().ref('/v2/groups/'+context.params.projectId+'/'+context.params.groupId+'/finishedCount')
     const groupRequiredCountRef = admin.database().ref('/v2/groups/'+context.params.projectId+'/'+context.params.groupId+'/requiredCount')

manager_dashboard/manager_dashboard/create.html

@@ -22,8 +22,24 @@
     <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.9/umd/popper.min.js" integrity="sha384-ApNbgh9B+Y1QKtv3Rn7W3mgPxhU9K/ScQsAP7hUibX39j7fakFPskvXusvfa0b4Q" crossorigin="anonymous"></script>
     <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>
     <link type="text/css" rel="stylesheet" href="css/mapswipe.css" />
+     <link rel="stylesheet" href="https://unpkg.com/[email protected]/dist/leaflet.css"
+   integrity="sha512-xwE/Az9zrjBIphAcBb3F6JVqxf46+CDLwfLMHloNu6KEQCAWi6HcDUbeOfBIptF7tcCzusKFjFw2yuvEpDL9wQ=="
+   crossorigin=""/>
+    <!-- Make sure you put this AFTER Leaflet's CSS -->
+ <script src="https://unpkg.com/[email protected]/dist/leaflet.js"
+   integrity="sha512-GffPMF3RvMeYyc1LWMHtK8EbPv0iNZ8/oTtHPx9/cc2ILxQ+u905qIwdpULaqDkyBKgOaB57QTMg7ztg8Jm2Og=="
+   crossorigin=""></script>
+    <script src='https://npmcdn.com/@turf/turf/turf.min.js'></script>
+
+    <script
+      src="https://browser.sentry-cdn.com/5.7.1/bundle.min.js"
+      integrity="sha384-KMv6bBTABABhv0NI+rVWly6PIRvdippFEgjpKyxUcpEmDWZTkDOiueL5xW+cztZZ"
+      crossorigin="anonymous"></script>
+    <script>
+        Sentry.init({ dsn: 'https://[email protected]/1788403' });
+    </script>
 </head>
-<body>
+<body onload="initMap()">
 
   <nav class="navbar navbar-expand-lg navbar-dark bg-dark">
     <a class="navbar-brand" href="index.html">
@@ -74,10 +90,30 @@ <h3>Basic Project Information</h3>
 
                       <form class="form-style-7">
                           <ul>
+                          <li>
+                              <label for="projectTopic">Project Topic</label>
+                              <input type="text" name="projectTopic" id="projectTopic" maxlength="50" required>
+                              <span>Enter the topic of your project (50 chars max).</span>
+                          </li>
+                          <li>
+                              <label for="projectRegion">Project Region</label>
+                              <input type="text" name="projectRegion" id="projectRegion" maxlength="50" required>
+                              <span>Enter name of your project Region (50 chars max).</span>
+                          </li>
+                          <li>
+                              <label for="projectNumber">Project Number</label>
+                              <input type="number" name="projectNumber" id="projectNumber" min="1" value="1">
+                              <span>Is this project part of a bigger campaign with multiple projects?</span>
+                          </li>
+                          <li>
+                              <label for="requestingOrganisation">Requesting Organisation</label>
+                              <input type="text" name="requestingOrganisation" id="requestingOrganisation" maxlength="100" required>
+                              <span>Which group, institution or community is requesting this project.</span>
+                          </li>
                           <li>
                               <label for="name">Name</label>
-                              <input type="text" name="name" id="name" maxlength="25" required>
-                              <span>Enter the name of your project (25 chars max).</span>
+                              <p id="name"></br>[projectTopic] - [projectRegion](taskNumber)<br>[requesting organisation]</p>
+                              <span>We will generate you project name based on your inputs above.</span>
                           </li>
                           <li>
                               <label for="lookFor">Look For</label>
@@ -113,7 +149,7 @@ <h3>Basic Project Information</h3>
                           </li>
                           <li>
                               <label for="groupSize">Group Size</label>
-                              <input type="number" name="groupSize" id="groupSize" value="50" min="10" max="200">
+                              <input type="number" name="groupSize" id="groupSize" value="120" min="10" max="200">
                               <span>How big should a mapping session be? Group size refers to the number of tasks per mapping session.</span>
                           </li>
                           <li></li>
@@ -136,8 +172,10 @@ <h3>Project Type Specific Information</h3>
                               <label for="geometry">GeoJSON File</label>
                               <!-- <textarea name="geometry" id="geometry" onkeyup="adjust_textarea(this)"></textarea> -->
                               <input type="file" accept=".GeoJSON" id="geometry" onchange='openFile(event)'>
+                              <div id="geometryMap"></div>
+                              <p id="geometryInfo"></p>
                               <p id="geometryContent"></p>
-                              <span>Upload your project area as GeoJSON File. Make sure that you provide a single polygon geometry.</span>
+                              <span>Upload your project area as GeoJSON File (max. 2.5MB). Make sure that you provide a single polygon geometry.</span>
                           </li>
                           <li>
                               <label for="tileServerBuildArea">Tile Server Name</label>
@@ -161,7 +199,7 @@ <h3>Project Type Specific Information</h3>
                           </li>
                           <li id="tileServerCreditsFieldBuildArea">
                               <label for="tileServerCreditsFieldBuildArea">Imagery Credits</label>
-                              <input type="text" name="tileServerCreditsBuildArea" id="tileServerCreditsBuildArea">
+                              <input type="text" name="tileServerCreditsBuildArea" id="tileServerCreditsBuildArea" value="© 2019 Microsoft Corporation, Earthstar Geographics SIO">
                               <span>Insert appropriate imagery credits. E.g. for Bing: "© 2019 Microsoft Corporation, Earthstar Geographics SIO"</span>
                           </li>
                           <li>
@@ -209,7 +247,7 @@ <h3>Project Type Specific Information</h3>
                           </li>
                           <li id="tileServerCreditsFieldFootprint">
                               <label for="tileServerCreditsFieldFootprint">Imagery Credits</label>
-                              <input type="text" name="tileServerCreditsFootprint" id="tileServerCreditsFootprint">
+                              <input type="text" name="tileServerCreditsFootprint" id="tileServerCreditsFootprint" value="© 2019 Microsoft Corporation, Earthstar Geographics SIO">
                               <span>Insert appropriate imagery credits. E.g. for Bing: "© 2019 Microsoft Corporation, Earthstar Geographics SIO"</span>
                           </li>
                           <li>
@@ -229,8 +267,10 @@ <h3>Project Type Specific Information</h3>
                               <label for="geometry">GeoJSON File</label>
                               <!-- <textarea name="geometry" id="geometry" onkeyup="adjust_textarea(this)"></textarea> -->
                               <input type="file" accept=".GeoJSON" id="geometryChangeDetection" onchange='openFile(event)'>
+                              <div id="geometryChangeDetectionMap"></div>
+                              <p id="geometryChangeDetectionInfo"></p>
                               <p id="geometryChangeDetectionContent"></p>
-                              <span>Upload your project area as GeoJSON File. Make sure that you provide a single polygon geometry.</span>
+                              <span>Upload your project area as GeoJSON File (max. 2.5MB). Make sure that you provide a single polygon geometry.</span>
                           </li>
                           <h4>Tile Server A</h4>
                           <li>

manager_dashboard/manager_dashboard/css/mapswipe.css

@@ -132,3 +132,6 @@
 	background: #6B9FFF;
 	color:#fff;
 }
+
+#geometryMap { height: 300px; width: 100%}
+#geometryChangeDetectionMap { height: 300px; width: 100%}

manager_dashboard/manager_dashboard/index.html

@@ -21,6 +21,15 @@
     <script src="https://code.jquery.com/jquery-3.2.1.slim.min.js" integrity="sha384-KJ3o2DKtIkvYIK3UENzmM7KCkRr/rE9/Qpg6aAZGJwFDMVNA/GpGFF93hXpG5KkN" crossorigin="anonymous"></script>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.9/umd/popper.min.js" integrity="sha384-ApNbgh9B+Y1QKtv3Rn7W3mgPxhU9K/ScQsAP7hUibX39j7fakFPskvXusvfa0b4Q" crossorigin="anonymous"></script>
     <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>
+
+    <script
+      src="https://browser.sentry-cdn.com/5.7.1/bundle.min.js"
+      integrity="sha384-KMv6bBTABABhv0NI+rVWly6PIRvdippFEgjpKyxUcpEmDWZTkDOiueL5xW+cztZZ"
+      crossorigin="anonymous"></script>
+    <script>
+        Sentry.init({ dsn: 'https://[email protected]/1788403' });
+    </script>
+
 </head>
 <body>