Update usergroups aggragated calc function.

Author: thenav56

django/apps/aggregated/management/commands/update_aggregated_data.py

@@ -109,8 +109,9 @@
         WITH used_tasks as (
             SELECT
               project_id, group_id, task_id
-            From results_user_groups ug
-                INNER JOIN results R USING (project_id, group_id, user_id)
+            From mapping_sessions_user_groups MSUR
+                INNER JOIN mapping_sessions MS USING (mapping_session_id)
+                INNER JOIN mapping_sessions_results MSR USING (mapping_session_id)
                 INNER JOIN tasks T USING (project_id, group_id, task_id)
             WHERE
                 R.timestamp >= %(from_date)s and R.timestamp < %(until_date)s
@@ -129,17 +130,18 @@
         -- Aggregate data by group
         user_group_data as (
             SELECT
-                ug.project_id,
-                ug.group_id,
-                ug.user_id,
-                ug.user_group_id,
-                MAX(R.timestamp::date) as timestamp_date,
-                MIN(R.start_time) as start_time,
-                MAX(R.end_time) as end_time,
-                COUNT(DISTINCT R.task_id) as task_count,
+                MS.project_id,
+                MS.group_id,
+                MS.user_id,
+                MSUR.user_group_id,
+                MAX(MS.start_time::date) as timestamp_date,
+                MIN(MS.start_time) as start_time,
+                MAX(MS.end_time) as end_time,
+                COUNT(DISTINCT T.task_id) as task_count,
                 SUM(T.area) as area_swiped
-            From results_user_groups ug
-                INNER JOIN results R USING (project_id, group_id, user_id)
+            From mapping_sessions_user_groups MSUR
+                INNER JOIN mapping_sessions MS USING (mapping_session_id)
+                INNER JOIN mapping_sessions_results MSR USING (mapping_session_id)
                 INNER JOIN task_data T USING (task_id)
             WHERE
                 R.timestamp >= %(from_date)s and R.timestamp < %(until_date)s

mapswipe_workers/mapswipe_workers/firebase_to_postgres/transfer_results.py

@@ -476,8 +476,12 @@ def save_user_group_results_to_postgres(
         p_con.query(filter_query, {"project_id": project_id})
 
     query_insert_results = """
-        INSERT INTO results_user_groups
-            SELECT * FROM results_user_groups_temp
+        INSERT INTO mapping_sessions_user_groups
+            SELECT
+                ms.mapping_session_id,
+                rug_temp.user_group_id
+            FROM results_user_groups_temp rug_temp
+                JOIN mapping_sessions ms USING (project_id, group_id, user_id)
         ON CONFLICT (project_id, group_id, user_id, user_group_id)
         DO NOTHING;
     """

mapswipe_workers/tests/integration/base.py

@@ -57,7 +57,7 @@ def _clear_all_data(cls):
                 TRUNCATE TABLE users_temp;
                 TRUNCATE TABLE user_groups_membership_logs_temp;
                 -- normal tables
-                TRUNCATE TABLE results_user_groups CASCADE;
+                TRUNCATE TABLE mapping_sessions_user_groups CASCADE;
                 TRUNCATE TABLE tasks CASCADE;
                 TRUNCATE TABLE user_groups_user_memberships CASCADE;
                 TRUNCATE TABLE user_groups CASCADE;

mapswipe_workers/tests/integration/set_up_db.sql

@@ -153,18 +153,11 @@ CREATE TABLE IF NOT EXISTS user_groups_user_memberships_temp (
 );
 
 -- Used to group results by user groups
-CREATE TABLE IF NOT EXISTS results_user_groups (
-    -- result primary key (not using task_id as it is a flat field in results)
-    project_id varchar,
-    group_id varchar,
-    user_id varchar,
-    -- user group primary key
-    user_group_id varchar,
-    PRIMARY KEY (project_id, group_id, user_id, user_group_id),
-    FOREIGN KEY (project_id) REFERENCES projects (project_id),
-    FOREIGN KEY (user_id) REFERENCES users (user_id),
-    FOREIGN KEY (user_group_id) REFERENCES user_groups (user_group_id),
-    FOREIGN KEY (project_id, group_id) REFERENCES groups (project_id, group_id)
+CREATE TABLE IF NOT EXISTS mapping_sessions_user_groups (
+    mapping_session_id int8,
+    user_group_id varchar,  -- user group primary key
+    PRIMARY KEY (mapping_session_id, user_group_id),
+    FOREIGN KEY (mapping_session_id) REFERENCES mapping_sessions (mapping_session_id),
 );
 
 -- create table for user_group_results import through csv

mapswipe_workers/tests/integration/tear_down.py

@@ -59,7 +59,7 @@ def delete_test_data(project_id: str) -> None:
     pg_db.query(sql_query, [project_id])
     sql_query = "DELETE FROM mapping_sessions WHERE project_id = %s"
     pg_db.query(sql_query, [project_id])
-    sql_query = "DELETE FROM results_user_groups WHERE project_id = %s"
+    sql_query = "DELETE FROM mapping_sessions_user_groups WHERE project_id = %s"
     pg_db.query(sql_query, [project_id])
     sql_query = "DELETE FROM results_temp WHERE project_id = %s"
     pg_db.query(sql_query, [project_id])

mapswipe_workers/tests/integration/test_transfer_results.py

@@ -203,7 +203,7 @@ def test_results_with_user_groups(self):
 
         UG_QUERY = "SELECT user_group_id FROM user_groups ORDER BY user_group_id"
         RUG_QUERY = (
-            "SELECT user_group_id FROM results_user_groups ORDER BY user_group_id"
+            "SELECT user_group_id FROM mapping_sessions_user_groups ORDER BY user_group_id"
         )
         for query, expected_value in [
             (

postgres/initdb.sql

@@ -153,18 +153,11 @@ CREATE TABLE IF NOT EXISTS user_groups_user_memberships_temp (
 );
 
 -- Used to group results by user groups
-CREATE TABLE IF NOT EXISTS results_user_groups (
-    -- result primary key (not using task_id as it is a flat field in results)
-    project_id varchar,
-    group_id varchar,
-    user_id varchar,
-    -- user group primary key
-    user_group_id varchar,
-    PRIMARY KEY (project_id, group_id, user_id, user_group_id),
-    FOREIGN KEY (project_id) REFERENCES projects (project_id),
-    FOREIGN KEY (user_id) REFERENCES users (user_id),
-    FOREIGN KEY (user_group_id) REFERENCES user_groups (user_group_id),
-    FOREIGN KEY (project_id, group_id) REFERENCES groups (project_id, group_id)
+CREATE TABLE IF NOT EXISTS mapping_sessions_user_groups (
+    mapping_session_id int8,
+    user_group_id varchar,  -- user group primary key
+    PRIMARY KEY (mapping_session_id, user_group_id),
+    FOREIGN KEY (mapping_session_id) REFERENCES mapping_sessions (mapping_session_id),
 );
 
 -- create table for user_group_results import through csv
@@ -223,7 +216,6 @@ BEGIN
         'Tried to insert invalid result: Project: % Group: % Task: % - User: %', v.project_id, v.group_id, NEW.task_id, v.user_id
             USING ERRCODE = '23503';
     END IF;
-
     RETURN NEW;
 END;
 $$;

postgres/scripts/v2_to_v3/05_update_results_user_groups_table.sql

@@ -0,0 +1,31 @@
+/*
+ * This script updates and tra the data from the 'results' table
+ * and transfers them to the new tables
+ * - 'mapping_sessions' and
+ * - 'mapping_sessions_results'
+ */
+SET search_path = 'public';
+
+-- Create new user_groups mapping sessions table
+CREATE TABLE IF NOT EXISTS mapping_sessions_user_groups (
+    mapping_session_id int8,
+    user_group_id varchar,
+    PRIMARY KEY (mapping_session_id, user_group_id),
+    FOREIGN KEY (mapping_session_id) REFERENCES mapping_sessions (mapping_session_id),
+    FOREIGN KEY (user_group_id) REFERENCES user_groups (user_group_id)
+);
+
+
+-- Copy data to new table
+INSERT INTO mapping_sessions_user_groups
+(
+    SELECT
+        MS.mapping_session_id,
+        rug.user_group_id
+    FROM results_user_groups rug
+        INNER JOIN mapping_sessions MS USING (project_id, group_id, user_id)
+)
+ON CONFLICT do nothing;
+
+-- NOTE: Drop old table (Do this manually after checking if all data is transferred)
+-- DROP TABLE IF EXISTS results_user_groups;