set progress and contributor count during transfer results process and not in generate stats

Author: Hagellach37

mapswipe_workers/mapswipe_workers/firebase_to_postgres/update_data.py

@@ -122,6 +122,72 @@ def update_project_data(project_ids: list = []):
         # if project.status is not existent
         data_update_project = [project.get("status", ""), project_id]
         pg_db.query(query_update_project, data_update_project)
-        logger.info(f"Updated status for project {project_id} in postgres")
 
     logger.info("Updated project data in Postgres")
+
+
+def set_progress_in_firebase(project_id: str):
+    """Update the project progress value in Firebase."""
+
+    pg_db = auth.postgresDB()
+    query = """
+        select
+          project_id
+          ,avg(group_progress)::integer as progress
+        from
+        (
+            select
+              g.group_id
+              ,g.project_id
+              ,case
+                when group_progress is null then 0
+                else group_progress
+              end as group_progress
+            from groups g
+            left join 
+                (
+                select
+                  r.group_id
+                  ,r.project_id
+                  ,case
+                    when count(distinct user_id) >= p.verification_number then 100
+                    else 100* count(distinct user_id) / p.verification_number
+                  end as group_progress
+                from results r, projects p
+                where r.project_id = p.project_id 
+                group by group_id, r.project_id, p.verification_number 
+            ) bar
+            on bar.group_id = g.group_id and bar.project_id = g.project_id
+            where g.project_id = %s 
+        ) foo	
+        group by project_id 
+    """
+    data = [project_id]
+    progress = pg_db.retr_query(query, data)[0][1]
+
+    fb_db = auth.firebaseDB()
+    project_progress_ref = fb_db.reference(f"v2/projects/{project_id}/progress")
+    project_progress_ref.set(progress)
+    logger.info(f"set progress for project {project_id}: {progress}")
+
+
+def set_contributor_count_in_firebase(project_id: str):
+    """Update the project contributors value in Firebase."""
+
+    pg_db = auth.postgresDB()
+    query = """
+        select
+          project_id
+          ,count(distinct user_id) contributor_count
+        from results r 
+        where 
+          project_id = %s 
+        group by project_id 
+    """
+    data = [project_id]
+    contributor_count = pg_db.retr_query(query, data)[0][1]
+
+    fb_db = auth.firebaseDB()
+    project_progress_ref = fb_db.reference(f"v2/projects/{project_id}/contributorCount")
+    project_progress_ref.set(contributor_count)
+    logger.info(f"set contributorCount attribute for project {project_id}: {contributor_count}")

mapswipe_workers/mapswipe_workers/generate_stats/project_stats.py

@@ -320,24 +320,6 @@ def get_agg_results_by_task_id(
     return agg_results_df
 
 
-def set_progress_in_firebase(project_id: str, progress: int):
-    """Update the project progress value in Firebase."""
-
-    fb_db = auth.firebaseDB()
-    project_progress_ref = fb_db.reference(f"v2/projects/{project_id}/progress")
-    project_progress_ref.set(progress)
-    logger.info(f"set progress for project {project_id}: {progress}")
-
-
-def set_contributor_count_in_firebase(project_id: str, contributor_count: int):
-    """Update the project contributors value in Firebase."""
-
-    fb_db = auth.firebaseDB()
-    project_progress_ref = fb_db.reference(f"v2/projects/{project_id}/contributorCount")
-    project_progress_ref.set(contributor_count)
-    logger.info(f"set contributorCount attribute for project {project_id}: {contributor_count}")
-
-
 def get_per_project_statistics(project_id: str, project_info: pd.Series) -> dict:
     """
     The function calculates all project related statistics.
@@ -416,21 +398,6 @@ def get_per_project_statistics(project_id: str, project_info: pd.Series) -> dict
             f"{project_stats_by_date_filename}"
         )
 
-        # update progress and contributors in firebase
-        # cum_progress[-1] refers to the latest value available for project progress
-        # cum_progess contains overall project progress reached at each day
-        set_progress_in_firebase(
-            project_id=project_id,
-            progress=int(100*project_stats_by_date_df["cum_progress"][-1])
-        )
-
-        # cum_number_of_users[-1] refers to the latest value available for contributorCount
-        # cum_number_of_users contains overall contributor count reached at each day
-        set_contributor_count_in_firebase(
-            project_id=project_id,
-            contributor_count=int(project_stats_by_date_df["cum_number_of_users"][-1])
-        )
-
         # generate geometries for HOT Tasking Manager
         if project_info.iloc[0]["project_type"] in [ProjectType.FOOTPRINT.value]:
             # do not do this for ArbitraryGeometry / BuildingFootprint projects

mapswipe_workers/mapswipe_workers/mapswipe_workers.py

@@ -115,6 +115,8 @@ def run_firebase_to_postgres() -> list:
     update_data.update_project_data()
     project_ids = transfer_results.transfer_results()
     for project_id in project_ids:
+        update_data.set_progress_in_firebase(project_id)
+        update_data.set_contributor_count_in_firebase(project_id)
         send_progress_notification(project_id)
     return project_ids