Merge pull request #190 from mapswipe/dev

fix write size limit

Author: Hagellach37

docs/source/index.rst

@@ -28,6 +28,7 @@ Welcome to MapSwipe Back-End's documentation!
 
    configuration
    installation
+   tutorials
    updating
    testing
    debugging

docs/source/tutorials.md

@@ -1,6 +1,18 @@
 # Tutorials
 For each project type there should be at least one tutorial. Tutorials are similar to actual projects. To display the tutorial we will use the same design and screens as if a user would map for real.
 
+## Deploy tutorials to Firebase Database from Docker
+You can upload tutorial data to Firebase like this:
+
+```bash
+docker run --name mapswipe_workers_local -it pythonmapswipeworkers_mapswipe_workers  bash`
+mapswipe_workers --verbose create-tutorial --input_file=sample_data/build_area_tutorial.json
+exit
+docker rm mapswipe_workers_local
+```
+
+If you want to use a customized tutorial make sure to adjust the `.json` file in the sample data folder and also add the file for the tasks.
+
 ## Data Perspective
 Tutorials and projects have the following in common:
 * both have groups with the same structure (there can also be several groups per tutorial)

mapswipe_workers/mapswipe_workers/base/base_project.py

@@ -8,6 +8,7 @@
 from mapswipe_workers import auth
 from mapswipe_workers.definitions import DATA_PATH
 from mapswipe_workers.definitions import logger
+from mapswipe_workers.definitions import CustomError
 
 
 class BaseProject(metaclass=ABCMeta):
@@ -137,22 +138,24 @@ def save_project(self, fb_db):
                         f' to firebase'
                         )
                 return True
-            except Exception:
+            except Exception as e:
+                self.delete_from_postgres()
                 logger.exception(
                         f'{self.projectId}'
                         f' - the project could not be saved'
                         f' to firebase. '
                         )
-                self.delete_from_postgres()
-                return False
-        except Exception:
+
+                logger.info(f'{self.projectId} deleted project data from postgres')
+                raise CustomError(e)
+        except Exception as e:
             logger.exception(
                     f'{self.projectId}'
                     f' - the project could not be saved'
                     f' to postgres and will therefor not be '
                     f' saved to firebase'
                     )
-            return False
+            raise CustomError(e)
 
     def save_to_firebase(self, fb_db, project, groups, groupsOfTasks):
 
@@ -164,16 +167,38 @@ def save_to_firebase(self, fb_db, project, groups, groupsOfTasks):
 
 
         ref = fb_db.reference('')
+        # save project
         ref.update({
-            f'v2/projects/{self.projectId}': project,
-            f'v2/groups/{self.projectId}': groups,
-            f'v2/tasks/{self.projectId}': groupsOfTasks,
+            f'v2/projects/{self.projectId}': project
             })
         logger.info(
                 f'{self.projectId} -'
-                f' uploaded project, groups and'
-                f' tasks to firebase realtime database'
+                f' uploaded project to firebase realtime database'
                 )
+        # save groups
+        ref.update({
+            f'v2/groups/{self.projectId}': groups
+        })
+        logger.info(
+            f'{self.projectId} -'
+            f' uploaded groups to firebase realtime database'
+        )
+        # save tasks, to avoid firebase write size limit we write chunks of task
+        # we write the tasks for 250 groups at once
+        task_upload_dict = {}
+
+        logger.info(f'there are {len(groupsOfTasks)} groups for this project')
+        for group_id, tasks_list in groupsOfTasks.items():
+            task_upload_dict[f'v2/tasks/{self.projectId}/{group_id}'] = tasks_list
+
+            if len(task_upload_dict) % 150 == 0:
+                ref.update(task_upload_dict)
+                logger.info(
+                    f'{self.projectId} -'
+                    f' uploaded 150 groups with tasks to firebase realtime database'
+                )
+                task_upload_dict = {}
+
         ref = fb_db.reference(f'v2/projectDrafts/{self.projectId}')
         ref.set({})
 

mapswipe_workers/mapswipe_workers/firebase_to_postgres/update_data.py

@@ -32,7 +32,6 @@ def update_user_data(user_ids=None):
         # No users in the Postgres database yet.
         # Get all users from Firebase.
         users = fb_ref.get()
-        print(users)
     else:
         # Get only new users from Firebase.
         last_updated = last_updated.strftime('%Y-%m-%dT%H:%M:%S.%fZ')

mapswipe_workers/mapswipe_workers/mapswipe_workers.py

@@ -348,7 +348,7 @@ def _run_create_projects(project_draft_ids=None):
                 project.calc_required_results()
                 if project.save_project(fb_db):
                     created_project_ids.append(project.projectId)
-                    newline = '\nnewline'
+                    newline = '\n'
                     message = (
                             f'### PROJECT CREATION SUCCESSFUL ###{newline}'
                             f'Project Name: {project.name}{newline}'

mapswipe_workers/sample_data/build_area_project_drafts.json

@@ -32,5 +32,27 @@
           "wmtsLayerName": "FALSE_COLOR"
         },
         "zoomLevel": 14
+    },
+    "build_area_big": {
+        "createdBy": "Sample Manager",
+        "geometry": {
+"type": "FeatureCollection",
+"name": "Bukindon_South",
+"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } },
+"features": [
+{ "type": "Feature", "properties": { }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 124.552337022305508, 8.013183564718995 ], [ 125.377278964821471, 8.029047832844302 ], [ 125.37539673, 8.02042961000001 ], [ 125.38337708, 8.00566959 ], [ 125.38165283, 7.98554993 ], [ 125.40270996, 7.96005011 ], [ 125.40054321, 7.93393993 ], [ 125.42089081, 7.91193008 ], [ 125.41744232, 7.81731987000001 ], [ 125.44413757, 7.75562 ], [ 125.44290924, 7.71647978 ], [ 125.42626953, 7.68472004 ], [ 125.43073273, 7.65559006 ], [ 125.42360687, 7.64100981 ], [ 125.39592743, 7.62140989 ], [ 125.37480164, 7.63410997 ], [ 125.35057831, 7.63816023 ], [ 125.32256317, 7.62744999 ], [ 125.30198669, 7.63472986 ], [ 125.30090332, 7.64885998 ], [ 125.28074646, 7.64744997 ], [ 125.28038025, 7.64074993 ], [ 125.25241089, 7.64373016000002 ], [ 125.25203705, 7.57984018000001 ], [ 125.21894073, 7.59195995 ], [ 125.21656799, 7.57053995 ], [ 125.19329834, 7.57212019 ], [ 125.16586304, 7.5229702 ], [ 125.15531921, 7.46537018 ], [ 125.13262939, 7.46412992 ], [ 125.11971283, 7.45313978000001 ], [ 125.06778717, 7.45175982000001 ], [ 125.05996704, 7.41956997 ], [ 125.04620361, 7.40458012000001 ], [ 124.97190857, 7.40071011000001 ], [ 124.94645691, 7.41130018 ], [ 124.92875671, 7.43205023 ], [ 124.89972687, 7.41360998 ], [ 124.88018036, 7.41554022 ], [ 124.87835693, 7.43260002 ], [ 124.86456299, 7.4320302 ], [ 124.84023285, 7.45099020000001 ], [ 124.84063721, 7.47724009 ], [ 124.82692719, 7.48998022000001 ], [ 124.81472015, 7.49597979000001 ], [ 124.7882309, 7.48022985 ], [ 124.79981232, 7.51733017 ], [ 124.82507324, 7.54148006 ], [ 124.82501221, 7.55834007 ], [ 124.81430054, 7.56812 ], [ 124.82051086, 7.57296991 ], [ 124.81165314, 7.57566977 ], [ 124.81788635, 7.59229994 ], [ 124.80444336, 7.5985198 ], [ 124.8025589, 7.62318993 ], [ 124.75379944, 7.65322018000001 ], [ 124.75311279, 7.67986012 ], [ 124.74120331, 7.68281984 ], [ 124.74056244, 7.69629002000001 ], [ 124.72891235, 7.70124006 ], [ 124.72543335, 7.7186799 ], [ 124.73486328, 7.73115015 ], [ 124.70761108, 7.75507021 ], [ 124.70349884, 7.77635002 ], [ 124.68052673, 7.79621983 ], [ 124.66120911, 7.83530998000002 ], [ 124.63118744, 7.84638977 ], [ 124.62483215, 7.87036991 ], [ 124.61730957, 7.87075996 ], [ 124.62412262, 7.89082003 ], [ 124.59819794, 7.90516996000001 ], [ 124.60048676, 7.9162302 ], [ 124.55732727, 7.9840498 ], [ 124.56124115, 7.99703979000002 ], [ 124.552337022305508, 8.013183564718995 ] ] ] ] } }
+]
+},
+        "image": "http://www.redcrosseth.org/media/k2/items/cache/5a05a447acfdf6fcc40548cc4c1cea8d_L.jpg",
+        "lookFor": "BUILDINGS",
+        "name": "BuildArea Big",
+        "projectDetails": "This project uses Bing as the tile server and zoom level 18. This is the setup most MapSwipe projects used so far.",
+        "verificationNumber": 3,
+        "groupSize": 120,
+        "tileServer": {
+          "name": "bing",
+          "credits": "© 2019 Microsoft Corporation, Earthstar Geographics SIO"
+        },
+        "projectType": 1
     }
 }

mapswipe_workers/tests/test_01_create_projects.py

@@ -18,7 +18,7 @@ def create_project_manager(email, username, password):
 def load_sample_project_drafts():
     sample_data = {}
     test_dir = os.path.dirname(os.path.abspath(__file__))
-    sample_data_dir = os.path.join(test_dir, 'sample_data/')
+    sample_data_dir = os.path.join(test_dir, '../sample_data/')
     for sample_project_drafts_json in glob.glob(sample_data_dir + '*_drafts.json'):
         if sample_project_drafts_json == 'build_area_to_big_project_drafts_test.json':
             continue