Pre-calculate total geo area and time max limit for project groups

Author: thenav56

django/apps/aggregated/management/commands/update_aggregated_data.py

@@ -19,27 +19,70 @@
 # |1|00:00:00.208768|00:00:01.398161|00:00:28.951521|
 # |2|00:00:01.330297|00:00:06.076814|00:00:03.481192|
 # |3|00:00:02.092967|00:00:11.271081|00:00:06.045881|
-TASK_GROUP_METADATA_QUERY = f"""
-          SELECT
-              project_id,
-              group_id,
-              SUM(
-                ST_Area(geom::geography(GEOMETRY,4326)) / 1000000
-              ) as total_task_group_area, -- sqkm
-              (
-                CASE
-                  -- Using 95_percent value of existing data for each project_type
-                  WHEN UG.project_type = {Project.Type.BUILD_AREA.value} THEN 1.4
-                  WHEN UG.project_type = {Project.Type.COMPLETENESS.value} THEN 1.4
-                  WHEN UG.project_type = {Project.Type.CHANGE_DETECTION.value} THEN 11.2
-                  -- FOOTPRINT: Not calculated right now
-                  WHEN UG.project_type = {Project.Type.FOOTPRINT.value} THEN 6.1
-                  ELSE 1
-                END
-              ) * COUNT(*) as time_spent_max_allowed
-          FROM tasks T
-              INNER JOIN used_task_groups UG USING (project_id, group_id)
-          GROUP BY project_id, project_type, group_id
+UPDATE_PROJECT_GROUP_DATA = f"""
+    WITH to_calculate_groups AS (
+        SELECT
+            project_id,
+            group_id
+        FROM groups
+            WHERE
+                (project_id, group_id) in (
+                    SELECT
+                        MS.project_id,
+                        MS.group_id
+                    FROM mapping_sessions MS
+                    WHERE
+                        MS.start_time >= %(from_date)s
+                        AND MS.start_time < %(until_date)s
+                    GROUP BY MS.project_id, MS.group_id
+                ) AND
+                (
+                    total_area is NULL OR time_spent_max_allowed is NULL
+                )
+    ),
+    groups_data AS (
+        SELECT
+          T.project_id,
+          T.group_id,
+          SUM( -- sqkm
+            ST_Area(T.geom::geography(GEOMETRY,4326)) / 1000000
+          ) as total_task_group_area,
+          (
+            CASE
+              -- Using 95_percent value of existing data for each project_type
+              WHEN P.project_type = {Project.Type.BUILD_AREA.value} THEN 1.4
+              WHEN P.project_type = {Project.Type.COMPLETENESS.value} THEN 1.4
+              WHEN P.project_type = {Project.Type.CHANGE_DETECTION.value} THEN 11.2
+              -- FOOTPRINT: Not calculated right now
+              WHEN P.project_type = {Project.Type.FOOTPRINT.value} THEN 6.1
+              ELSE 1
+            END
+          ) * COUNT(*) as time_spent_max_allowed
+        FROM tasks T
+          INNER JOIN to_calculate_groups G USING (project_id, group_id)
+          INNER JOIN projects P USING (project_id)
+        GROUP BY project_id, P.project_type, group_id
+    )
+    UPDATE groups G
+    SET
+        total_area = GD.total_task_group_area,
+        time_spent_max_allowed = GD.time_spent_max_allowed
+    FROM groups_data GD
+    WHERE
+        G.project_id = GD.project_id AND
+        G.group_id = GD.group_id;
+"""
+
+TASK_GROUP_METADATA_QUERY = """
+        SELECT
+            G.project_id,
+            G.group_id,
+            G.total_area as total_task_group_area,
+            G.time_spent_max_allowed
+        FROM groups G
+            INNER JOIN used_task_groups UG USING (project_id, group_id)
+            INNER JOIN projects P USING (project_id)
+        GROUP BY G.project_id, P.project_type, G.group_id
 """
 
 
@@ -239,6 +282,20 @@ def _track(self, tracker_type, label, sql):
                 until_date=until_date.strftime("%Y-%m-%d"),
             )
             start_time = time.time()
+
+            self.stdout.write(
+                f"Updating Project Group Data for {label.title()} for date: {params}"
+            )
+            with transaction.atomic():
+                with connection.cursor() as cursor:
+                    cursor.execute(UPDATE_PROJECT_GROUP_DATA, params)
+                self.stdout.write(
+                    self.style.SUCCESS(
+                        f"Successfull. Runtime: {time.time() - start_time} seconds"
+                    )
+                )
+
+            start_time = time.time()
             self.stdout.write(f"Updating {label.title()} Data for date: {params}")
             with transaction.atomic():
                 with connection.cursor() as cursor:

django/apps/existing_database/models.py

@@ -106,6 +106,8 @@ class Group(Model):
     progress = models.IntegerField(blank=True, null=True)
     # Database uses JSON instead of JSONB (not supported by django)
     project_type_specifics = models.TextField(blank=True, null=True)
+    total_area = models.FloatField(blank=True, null=True)
+    time_spent_max_allowed = models.FloatField(blank=True, null=True)
 
     # Django derived fields from ForeignKey
     project_id: str

mapswipe_workers/tests/integration/set_up_db.sql

@@ -30,6 +30,9 @@ CREATE TABLE IF NOT EXISTS groups (
     required_count int,
     progress int,
     project_type_specifics json,
+    --  total_area & time_spent_max_allowed are maintaned and used by aggregated module
+    total_area float DEFAULT NULL,
+    time_spent_max_allowed float DEFAULT NULL,
     PRIMARY KEY (project_id, group_id),
     FOREIGN KEY (project_id) REFERENCES projects (project_id)
 );

postgres/initdb.sql

@@ -30,6 +30,9 @@ CREATE TABLE IF NOT EXISTS groups (
     required_count int,
     progress int,
     project_type_specifics json,
+    --  total_area & time_spent_max_allowed are maintaned and used by aggregated module
+    total_area float DEFAULT NULL,
+    time_spent_max_allowed float DEFAULT NULL,
     PRIMARY KEY (project_id, group_id),
     FOREIGN KEY (project_id) REFERENCES projects (project_id)
 );

postgres/scripts/backtrack_user_group_mapping_sessions.sql

@@ -55,10 +55,11 @@ INSERT INTO mapping_sessions_user_groups(
   WHERE MS.end_time >= '2022-01-01'
 ) ON CONFLICT (mapping_session_id, user_group_id) DO NOTHING;
 
+-- To update aggregated data: docker-compose exec django ./manage.py update_aggregated_data
 -- This table is maintaned using django
 -- type = 1 = USER_GROUP
+-- type = 0 = USER
 -- value = date from when the data is calculated >= 2022-01-01
--- To update aggregated data: docker-compose exec django ./manage.py update_aggregated_data.py
 UPDATE aggregated_aggregatedtracking
 SET "value" = '2022-01-01'
-WHERE "type" = 1;
+WHERE "type" in (0, 1);

postgres/scripts/v2_to_v3/06_alter_groups_data_add_total_area_and_time_limit_columns.sql

@@ -0,0 +1,5 @@
+
+--  total_area & time_spent_max_allowed are maintaned and used by aggregated module
+ALTER TABLE groups
+    ADD COLUMN total_area float DEFAULT NULL,
+    ADD COLUMN time_spent_max_allowed float DEFAULT NULL;