Skip to content

Commit 220784d

Browse files
Hagellach37Hagellach37
authored
Merge pull request #630 from mapswipe/dev
Dev - cap max time per mapping session for community dashboard
2 parents f12a789 + 06ec554 commit 220784d

File tree

1 file changed

+65
-42
lines changed

1 file changed

+65
-42
lines changed

‎django/apps/aggregated/management/commands/update_aggregated_data.py‎

Lines changed: 65 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,43 @@
66
AggregatedUserGroupStatData,
77
AggregatedUserStatData,
88
)
9-
from apps.existing_database.models import MappingSession
9+
from apps.existing_database.models import MappingSession, Project
1010
from django.core.management.base import BaseCommand
1111
from django.db import connection, models, transaction
1212
from django.utils import timezone
1313

14+
# Factor calculated by @Hagellach37
15+
# For defining the threshold for outliers using `95_percent`
16+
# Used by TASK_GROUP_METADATA_QUERY
17+
# |project_type|median|95_percent|avg|
18+
# |------------|------|----------|---|
19+
# |1|00:00:00.208768|00:00:01.398161|00:00:28.951521|
20+
# |2|00:00:01.330297|00:00:06.076814|00:00:03.481192|
21+
# |3|00:00:02.092967|00:00:11.271081|00:00:06.045881|
22+
TASK_GROUP_METADATA_QUERY = f"""
23+
SELECT
24+
project_id,
25+
group_id,
26+
SUM(
27+
ST_Area(geom::geography(GEOMETRY,4326)) / 1000000
28+
) as total_task_group_area, -- sqkm
29+
(
30+
CASE
31+
-- Using 95_percent value of existing data for each project_type
32+
WHEN UG.project_type = {Project.Type.BUILD_AREA.value} THEN 1.4
33+
WHEN UG.project_type = {Project.Type.COMPLETENESS.value} THEN 1.4
34+
WHEN UG.project_type = {Project.Type.CHANGE_DETECTION.value} THEN 11.2
35+
-- FOOTPRINT: Not calculated right now
36+
WHEN UG.project_type = {Project.Type.FOOTPRINT.value} THEN 6.1
37+
ELSE 1
38+
END
39+
) * COUNT(*) as time_spent_max_allowed
40+
FROM tasks T
41+
INNER JOIN used_task_groups UG USING (project_id, group_id)
42+
GROUP BY project_id, project_type, group_id
43+
"""
44+
45+
1446
UPDATE_USER_DATA_SQL = f"""
1547
INSERT INTO "{AggregatedUserStatData._meta.db_table}" (
1648
project_id,
@@ -26,49 +58,45 @@
2658
WITH used_task_groups as (
2759
SELECT
2860
MS.project_id,
61+
P.project_type,
2962
MS.group_id
3063
FROM mapping_sessions MS
3164
INNER JOIN projects P USING (project_id)
3265
WHERE
33-
MS.start_time >= %(from_date)s and MS.start_time < %(until_date)s
34-
AND P.project_type != 2 -- Skip for footprint type missions
35-
GROUP BY project_id, group_id -- To get unique
66+
-- Skip for footprint type missions
67+
P.project_type != {Project.Type.FOOTPRINT.value}
68+
AND MS.start_time >= %(from_date)s
69+
AND MS.start_time < %(until_date)s
70+
GROUP BY project_id, project_type, group_id -- To get unique
3671
),
3772
-- Calculated area by task_groups
38-
task_group_area_data as (
39-
SELECT
40-
project_id,
41-
group_id,
42-
SUM(
43-
ST_Area(geom::geography(GEOMETRY,4326)) / 1000000
44-
) as total_task_group_area -- sqkm
45-
FROM tasks T
46-
INNER JOIN used_task_groups UG USING (project_id, group_id)
47-
GROUP BY project_id, group_id
48-
),
73+
task_group_metadata as ({TASK_GROUP_METADATA_QUERY}),
4974
-- Aggregate data by user
5075
user_data as (
5176
SELECT
5277
MS.project_id,
5378
MS.group_id,
5479
MS.user_id,
5580
MS.start_time::date as timestamp_date,
56-
MS.start_time,
57-
MS.end_time,
81+
LEAST(
82+
EXTRACT(EPOCH FROM (MS.end_time - MS.start_time)),
83+
TG.time_spent_max_allowed
84+
) as time_spent_sec,
5885
MS.items_count as task_count,
5986
Coalesce(TG.total_task_group_area, 0) as area_swiped
6087
FROM mapping_sessions MS
61-
LEFT JOIN task_group_area_data TG USING (project_id, group_id)
88+
LEFT JOIN task_group_metadata TG USING (project_id, group_id)
6289
WHERE
63-
MS.start_time >= %(from_date)s and MS.start_time < %(until_date)s
90+
MS.start_time >= %(from_date)s
91+
AND MS.start_time < %(until_date)s
6492
),
6593
-- Additional aggregate by timestamp_date
6694
user_agg_data as (
6795
SELECT
6896
project_id,
6997
user_id,
7098
timestamp_date,
71-
COALESCE(SUM(EXTRACT(EPOCH FROM (end_time - start_time))), 0) as total_time,
99+
COALESCE(SUM(time_spent_sec), 0) as total_time,
72100
COALESCE(SUM(task_count), 0) as task_count,
73101
COALESCE(SUM(area_swiped), 0) as area_swiped
74102
FROM user_data
@@ -92,7 +120,6 @@
92120
swipes = EXCLUDED.swipes;
93121
"""
94122

95-
96123
UPDATE_USER_GROUP_SQL = f"""
97124
INSERT INTO "{AggregatedUserGroupStatData._meta.db_table}" (
98125
project_id,
@@ -109,27 +136,20 @@
109136
WITH used_task_groups as (
110137
SELECT
111138
MS.project_id,
139+
P.project_type,
112140
MS.group_id
113-
From mapping_sessions_user_groups MSUR
141+
FROM mapping_sessions_user_groups MSUR
114142
INNER JOIN mapping_sessions MS USING (mapping_session_id)
115143
INNER JOIN projects P USING (project_id)
116144
WHERE
117-
MS.start_time >= %(from_date)s and MS.start_time < %(until_date)s
118-
AND P.project_type != 2 -- Skip for footprint type missions
119-
GROUP BY project_id, group_id -- To get unique
145+
-- Skip for footprint type missions
146+
P.project_type != {Project.Type.FOOTPRINT.value}
147+
AND MS.start_time >= %(from_date)s
148+
AND MS.start_time < %(until_date)s
149+
GROUP BY project_id, project_type, group_id -- To get unique
120150
),
121151
-- Calculated area by task_groups
122-
task_group_area_data as (
123-
SELECT
124-
project_id,
125-
group_id,
126-
SUM(
127-
ST_Area(geom::geography(GEOMETRY,4326)) / 1000000
128-
) as total_task_group_area -- sqkm
129-
FROM tasks T
130-
INNER JOIN used_task_groups UG USING (project_id, group_id)
131-
GROUP BY project_id, group_id
132-
),
152+
task_group_metadata as ({TASK_GROUP_METADATA_QUERY}),
133153
-- Aggregate data by user-group
134154
user_group_data as (
135155
SELECT
@@ -138,15 +158,18 @@
138158
MS.user_id,
139159
MSUR.user_group_id,
140160
MS.start_time::date as timestamp_date,
141-
MS.start_time as start_time,
142-
MS.end_time as end_time,
161+
LEAST(
162+
EXTRACT(EPOCH FROM (MS.end_time - MS.start_time)),
163+
TG.time_spent_max_allowed
164+
) as time_spent_sec,
143165
MS.items_count as task_count,
144166
Coalesce(TG.total_task_group_area, 0) as area_swiped
145-
From mapping_sessions_user_groups MSUR
167+
FROM mapping_sessions_user_groups MSUR
146168
INNER JOIN mapping_sessions MS USING (mapping_session_id)
147-
LEFT JOIN task_group_area_data TG USING (project_id, group_id)
169+
LEFT JOIN task_group_metadata TG USING (project_id, group_id)
148170
WHERE
149-
MS.start_time >= %(from_date)s and MS.start_time < %(until_date)s
171+
MS.start_time >= %(from_date)s
172+
AND MS.start_time < %(until_date)s
150173
),
151174
-- Additional aggregate by timestamp_date
152175
user_group_agg_data as (
@@ -155,7 +178,7 @@
155178
user_id,
156179
user_group_id,
157180
timestamp_date,
158-
COALESCE(SUM(EXTRACT(EPOCH FROM (end_time - start_time))), 0) as total_time,
181+
COALESCE(SUM(time_spent_sec), 0) as total_time,
159182
COALESCE(SUM(task_count), 0) as task_count,
160183
COALESCE(SUM(area_swiped), 0) as area_swiped
161184
FROM user_group_data

0 commit comments

Comments
 (0)