Merge pull request #596 from mapswipe/dev

Dev - add generate stats

Author: Hagellach37

mapswipe_workers/mapswipe_workers/firebase_to_postgres/delete_project.py

@@ -22,6 +22,10 @@ def delete_project(project_ids: list) -> bool:
     Deletes project, groups, tasks and results from Firebase and Postgres.
     """
     for project_id in project_ids:
+        if project_id is None:
+            logger.info("Will not delete Null project_id.")
+            continue
+
         logger.info(
             f"Delete project, groups, tasks and results of project: {project_id}"
         )
@@ -90,7 +94,18 @@ def delete_project(project_ids: list) -> bool:
         ref.delete()
 
         pg_db = auth.postgresDB()
-        sql_query = "DELETE FROM results WHERE project_id = %(project_id)s;"
+        sql_query = """
+            DELETE FROM mapping_sessions_results msr
+            USING mapping_sessions ms
+            WHERE ms.mapping_session_id = msr.mapping_session_id
+                AND ms.project_id = %(project_id)s;
+        """
+        pg_db.query(sql_query, {"project_id": project_id})
+        sql_query = """
+            DELETE
+            FROM mapping_sessions
+            WHERE project_id = %(project_id)s ;
+        """
         pg_db.query(sql_query, {"project_id": project_id})
         sql_query = "DELETE FROM tasks WHERE project_id = %(project_id)s;"
         pg_db.query(sql_query, {"project_id": project_id})

mapswipe_workers/mapswipe_workers/firebase_to_postgres/update_data.py

@@ -567,21 +567,22 @@ def update_project_data(project_ids: list = []):
     logger.info("Finished status update projects.")
 
 
-def set_progress_in_firebase(project_id: str):
-    """Update the project progress value in Firebase."""
-
+def get_project_progress(project_id: str) -> int:
+    """
+    Calculate overall project progress as the average progress for all groups.
+    This is not hundred percent exact, since groups can have a different number of tasks
+    but it is still "good enough" and gives almost correct progress.
+    But it is easier to compute than considering the actual number of tasks per group.
+
+    NOTE: the cast to integer in postgres rounds decimals. This means that for 99.5%
+    progress, we return 100% here. We should evaluate if this is what we want if/when
+    we introduce automated project rotation upon completion (as the reported completion
+    would happen 0.5% before actual completion).
+    """
     pg_db = auth.postgresDB()
     query = """
-        -- Calculate overall project progress as
-        -- the average progress for all groups.
-        -- This is not hundred percent exact,
-        -- since groups can have a different number of tasks
-        -- but it is still "good enough" and gives almost correct progress.
-        -- But it is easier to compute
-        -- than considering the actual number of tasks per group.
         select
-          project_id
-          ,avg(group_progress)::integer as progress
+          avg(group_progress)::integer as progress
         from
         (
             -- Get all groups for this project and
@@ -605,46 +606,52 @@ def set_progress_in_firebase(project_id: str):
                 -- even if more users than required submitted results.
                 -- The verification number of a project is used here.
                 select
-                  r.group_id
-                  ,r.project_id
+                  ms.group_id
+                  ,ms.project_id
                   ,case
                     when count(distinct user_id) >= p.verification_number then 100
                     else 100 * count(distinct user_id) / p.verification_number
                   end as group_progress
-                from results r, projects p
-                where r.project_id = p.project_id
-                group by group_id, r.project_id, p.verification_number
+                from mapping_sessions ms, projects p
+                where ms.project_id = p.project_id
+                group by group_id, ms.project_id, p.verification_number
             ) bar
             on bar.group_id = g.group_id and bar.project_id = g.project_id
             where g.project_id = %s
         ) foo
         group by project_id
     """
     data = [project_id]
-    progress = pg_db.retr_query(query, data)[0][1]
+    return pg_db.retr_query(query, data)[0][0]
+
+
+def set_progress_in_firebase(project_id: str):
+    """Update the project progress value in Firebase."""
+    progress = get_project_progress(project_id)
 
     fb_db = auth.firebaseDB()
     project_progress_ref = fb_db.reference(f"v2/projects/{project_id}/progress")
     project_progress_ref.set(progress)
     logger.info(f"set progress for project {project_id}: {progress}")
 
 
-def set_contributor_count_in_firebase(project_id: str):
-    """Update the project contributors value in Firebase."""
-
+def get_contributor_count_from_postgres(project_id: str) -> int:
     pg_db = auth.postgresDB()
     query = """
         select
-          project_id
-          ,count(distinct user_id) contributor_count
-        from results r
+          count(distinct user_id)
+        from mapping_sessions ms
         where
           project_id = %s
-        group by project_id
     """
     data = [project_id]
-    contributor_count = pg_db.retr_query(query, data)[0][1]
+    return pg_db.retr_query(query, data)[0][0]
+
+
+def set_contributor_count_in_firebase(project_id: str):
+    """Update the project contributors value in Firebase."""
 
+    contributor_count = get_contributor_count_from_postgres(project_id)
     fb_db = auth.firebaseDB()
     project_progress_ref = fb_db.reference(f"v2/projects/{project_id}/contributorCount")
     project_progress_ref.set(contributor_count)

mapswipe_workers/mapswipe_workers/generate_stats/project_stats.py

@@ -104,8 +104,19 @@ def get_results(filename: str, project_id: str) -> pd.DataFrame:
     sql_query = sql.SQL(
         """
         COPY (
-            SELECT results.*, U.username
-            FROM results
+            SELECT
+                ms.project_id,
+                ms.group_id,
+                ms.user_id,
+                msr.task_id,
+                ms.start_time as timestamp,
+                ms.start_time,
+                ms.end_time,
+                msr.result,
+                U.username
+            FROM mapping_sessions_results msr
+            LEFT JOIN mapping_sessions ms ON
+                ms.mapping_session_id = msr.mapping_session_id
             LEFT JOIN users U USING (user_id)
             WHERE project_id = {}
         ) TO STDOUT WITH CSV HEADER

mapswipe_workers/tests/integration/base.py

@@ -57,7 +57,6 @@ def _clear_all_data(cls):
                 TRUNCATE TABLE users_temp;
                 TRUNCATE TABLE user_groups_membership_logs_temp;
                 -- normal tables
-                TRUNCATE TABLE results CASCADE;
                 TRUNCATE TABLE results_user_groups CASCADE;
                 TRUNCATE TABLE tasks CASCADE;
                 TRUNCATE TABLE user_groups_user_memberships CASCADE;

mapswipe_workers/tests/integration/fixtures/tile_map_service_grid/mapping_sessions/build_area.csv

@@ -0,0 +1 @@
+test_build_area	g115	test_build_area	123	2020-02-03 15:39:39.332	2020-02-03 15:39:42.332	252