work on write size limit #188

Author: Hagellach37

mapswipe_workers/mapswipe_workers/base/base_project.py

@@ -167,16 +167,41 @@ def save_to_firebase(self, fb_db, project, groups, groupsOfTasks):
 
 
         ref = fb_db.reference('')
+        # save project
         ref.update({
-            f'v2/projects/{self.projectId}': project,
-            f'v2/groups/{self.projectId}': groups,
-            f'v2/tasks/{self.projectId}': groupsOfTasks,
+            f'v2/projects/{self.projectId}': project
             })
         logger.info(
                 f'{self.projectId} -'
-                f' uploaded project, groups and'
-                f' tasks to firebase realtime database'
+                f' uploaded project to firebase realtime database'
                 )
+        # save groups
+        ref.update({
+            f'v2/groups/{self.projectId}': groups
+        })
+        logger.info(
+            f'{self.projectId} -'
+            f' uploaded groups to firebase realtime database'
+        )
+        # save tasks, to avoid firebase write size limit we write chunks of task
+        # we write the tasks for 250 groups at once
+        task_upload_dict = {}
+
+        logger.info(f'there are {len(groupsOfTasks)} groups for this project')
+        for group_id, tasks_list in groupsOfTasks.items():
+            task_upload_dict[f'v2/projects/{self.projectId}/{group_id}'] = tasks_list
+
+            if len(task_upload_dict) % 150 == 0:
+                logger.info(task_upload_dict)
+
+                ref.update(task_upload_dict)
+                logger.info(
+                    f'{self.projectId} -'
+                    f' uploaded 150 groups with tasks to firebase realtime database'
+                )
+                task_upload_dict = {}
+
+
         ref = fb_db.reference(f'v2/projectDrafts/{self.projectId}')
         ref.set({})
 

mapswipe_workers/sample_data/build_area_project_drafts.json

@@ -32,5 +32,27 @@
           "wmtsLayerName": "FALSE_COLOR"
         },
         "zoomLevel": 14
+    },
+    "build_area_big": {
+        "createdBy": "Sample Manager",
+        "geometry": {
+"type": "FeatureCollection",
+"name": "Bukindon_South",
+"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } },
+"features": [
+{ "type": "Feature", "properties": { }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 124.552337022305508, 8.013183564718995 ], [ 125.377278964821471, 8.029047832844302 ], [ 125.37539673, 8.02042961000001 ], [ 125.38337708, 8.00566959 ], [ 125.38165283, 7.98554993 ], [ 125.40270996, 7.96005011 ], [ 125.40054321, 7.93393993 ], [ 125.42089081, 7.91193008 ], [ 125.41744232, 7.81731987000001 ], [ 125.44413757, 7.75562 ], [ 125.44290924, 7.71647978 ], [ 125.42626953, 7.68472004 ], [ 125.43073273, 7.65559006 ], [ 125.42360687, 7.64100981 ], [ 125.39592743, 7.62140989 ], [ 125.37480164, 7.63410997 ], [ 125.35057831, 7.63816023 ], [ 125.32256317, 7.62744999 ], [ 125.30198669, 7.63472986 ], [ 125.30090332, 7.64885998 ], [ 125.28074646, 7.64744997 ], [ 125.28038025, 7.64074993 ], [ 125.25241089, 7.64373016000002 ], [ 125.25203705, 7.57984018000001 ], [ 125.21894073, 7.59195995 ], [ 125.21656799, 7.57053995 ], [ 125.19329834, 7.57212019 ], [ 125.16586304, 7.5229702 ], [ 125.15531921, 7.46537018 ], [ 125.13262939, 7.46412992 ], [ 125.11971283, 7.45313978000001 ], [ 125.06778717, 7.45175982000001 ], [ 125.05996704, 7.41956997 ], [ 125.04620361, 7.40458012000001 ], [ 124.97190857, 7.40071011000001 ], [ 124.94645691, 7.41130018 ], [ 124.92875671, 7.43205023 ], [ 124.89972687, 7.41360998 ], [ 124.88018036, 7.41554022 ], [ 124.87835693, 7.43260002 ], [ 124.86456299, 7.4320302 ], [ 124.84023285, 7.45099020000001 ], [ 124.84063721, 7.47724009 ], [ 124.82692719, 7.48998022000001 ], [ 124.81472015, 7.49597979000001 ], [ 124.7882309, 7.48022985 ], [ 124.79981232, 7.51733017 ], [ 124.82507324, 7.54148006 ], [ 124.82501221, 7.55834007 ], [ 124.81430054, 7.56812 ], [ 124.82051086, 7.57296991 ], [ 124.81165314, 7.57566977 ], [ 124.81788635, 7.59229994 ], [ 124.80444336, 7.5985198 ], [ 124.8025589, 7.62318993 ], [ 124.75379944, 7.65322018000001 ], [ 124.75311279, 7.67986012 ], [ 124.74120331, 7.68281984 ], [ 124.74056244, 7.69629002000001 ], [ 124.72891235, 7.70124006 ], [ 124.72543335, 7.7186799 ], [ 124.73486328, 7.73115015 ], [ 124.70761108, 7.75507021 ], [ 124.70349884, 7.77635002 ], [ 124.68052673, 7.79621983 ], [ 124.66120911, 7.83530998000002 ], [ 124.63118744, 7.84638977 ], [ 124.62483215, 7.87036991 ], [ 124.61730957, 7.87075996 ], [ 124.62412262, 7.89082003 ], [ 124.59819794, 7.90516996000001 ], [ 124.60048676, 7.9162302 ], [ 124.55732727, 7.9840498 ], [ 124.56124115, 7.99703979000002 ], [ 124.552337022305508, 8.013183564718995 ] ] ] ] } }
+]
+},
+        "image": "http://www.redcrosseth.org/media/k2/items/cache/5a05a447acfdf6fcc40548cc4c1cea8d_L.jpg",
+        "lookFor": "BUILDINGS",
+        "name": "BuildArea Big",
+        "projectDetails": "This project uses Bing as the tile server and zoom level 18. This is the setup most MapSwipe projects used so far.",
+        "verificationNumber": 3,
+        "groupSize": 120,
+        "tileServer": {
+          "name": "bing",
+          "credits": "© 2019 Microsoft Corporation, Earthstar Geographics SIO"
+        },
+        "projectType": 1
     }
 }

mapswipe_workers/tests/test_01_create_projects.py

@@ -18,7 +18,7 @@ def create_project_manager(email, username, password):
 def load_sample_project_drafts():
     sample_data = {}
     test_dir = os.path.dirname(os.path.abspath(__file__))
-    sample_data_dir = os.path.join(test_dir, 'sample_data/')
+    sample_data_dir = os.path.join(test_dir, '../sample_data/')
     for sample_project_drafts_json in glob.glob(sample_data_dir + '*_drafts.json'):
         if sample_project_drafts_json == 'build_area_to_big_project_drafts_test.json':
             continue