Skip to content

Commit 2f56330

Browse files
thenav56thenav56
committed
Upgrade aggregate data inject logic.
- Calculating area by mapping session.
1 parent 78bc1a4 commit 2f56330

File tree

2 files changed

+58
-65
lines changed

2 files changed

+58
-65
lines changed

‎django/apps/aggregated/management/commands/update_aggregated_data.py‎

Lines changed: 42 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -22,47 +22,44 @@
2222
swipes
2323
)
2424
(
25-
-- Retrieve used tasks
26-
-- task_id is not the primary key. project_id-group_id-task_id is
27-
WITH used_tasks as (
25+
-- Retrieve used task groups
26+
WITH used_task_groups as (
2827
SELECT
29-
MS.project_id, MS.group_id, MSR.task_id
30-
FROM mapping_sessions_results MSR
31-
INNER JOIN mapping_sessions MS USING (mapping_session_id)
32-
INNER JOIN tasks T USING (project_id, group_id, task_id)
28+
MS.project_id, MS.group_id
29+
FROM mapping_sessions MS
3330
WHERE
3431
MS.start_time >= %(from_date)s and MS.start_time < %(until_date)s
35-
GROUP BY project_id, group_id, task_id
32+
GROUP BY project_id, group_id -- To get unique
3633
),
37-
-- Calculated task area.
38-
task_data as (
34+
-- Calculated area by task_groups
35+
task_group_data as (
3936
SELECT
4037
project_id,
4138
group_id,
42-
task_id,
43-
ST_Area(geom::geography(GEOMETRY,4326)) / 1000000 as area -- sqkm
44-
FROM used_tasks
45-
INNER JOIN tasks T USING (project_id, group_id, task_id)
39+
SUM(
40+
ST_Area(geom::geography(GEOMETRY,4326)) / 1000000
41+
) as total_task_group_area -- sqkm
42+
FROM tasks T
43+
INNER JOIN used_task_groups UG USING (project_id, group_id)
44+
GROUP BY project_id, group_id
4645
),
47-
-- Aggregate data by group
46+
-- Aggregate data by user
4847
user_data as (
4948
SELECT
5049
MS.project_id,
5150
MS.group_id,
5251
MS.user_id,
53-
MAX(MS.start_time::date) as timestamp_date,
54-
MIN(MS.start_time) as start_time,
55-
MAX(MS.end_time) as end_time,
56-
COUNT(DISTINCT MSR.task_id) as task_count,
57-
SUM(T.area) as area_swiped
58-
FROM mapping_sessions_results MSR
59-
INNER JOIN mapping_sessions MS USING (mapping_session_id)
60-
INNER JOIN task_data T USING (project_id, group_id, task_id)
52+
MS.start_time::date as timestamp_date,
53+
MS.start_time,
54+
MS.end_time,
55+
MS.items_count as task_count,
56+
TG.total_task_group_area as area_swiped
57+
FROM mapping_sessions MS
58+
INNER JOIN task_group_data TG USING (project_id, group_id)
6159
WHERE
6260
MS.start_time >= %(from_date)s and MS.start_time < %(until_date)s
63-
GROUP BY MS.project_id, MS.group_id, MS.user_id
6461
),
65-
-- Aggregate group data
62+
-- Additional aggregate by timestamp_date
6663
user_agg_data as (
6764
SELECT
6865
project_id,
@@ -105,49 +102,47 @@
105102
swipes
106103
)
107104
(
108-
-- Retrieve used tasks
109-
WITH used_tasks as (
105+
-- Retrieve used task groups
106+
WITH used_task_groups as (
110107
SELECT
111-
project_id, group_id, task_id
108+
MS.project_id, MS.group_id
112109
From mapping_sessions_user_groups MSUR
113110
INNER JOIN mapping_sessions MS USING (mapping_session_id)
114-
INNER JOIN mapping_sessions_results MSR USING (mapping_session_id)
115-
INNER JOIN tasks T USING (project_id, group_id, task_id)
116111
WHERE
117112
MS.start_time >= %(from_date)s and MS.start_time < %(until_date)s
118-
GROUP BY project_id, group_id, task_id
113+
GROUP BY project_id, group_id -- To get unique
119114
),
120-
-- Calculated task area.
121-
task_data as (
115+
-- Calculated area by task_groups
116+
task_group_data as (
122117
SELECT
123118
project_id,
124119
group_id,
125-
task_id,
126-
ST_Area(geom::geography(GEOMETRY,4326)) / 1000000 as area -- sqkm
127-
FROM used_tasks
128-
INNER JOIN tasks T USING (project_id, group_id, task_id)
120+
SUM(
121+
ST_Area(geom::geography(GEOMETRY,4326)) / 1000000
122+
) as total_task_group_area -- sqkm
123+
FROM tasks T
124+
INNER JOIN used_task_groups UG USING (project_id, group_id)
125+
GROUP BY project_id, group_id
129126
),
130-
-- Aggregate data by group
127+
-- Aggregate data by user-group
131128
user_group_data as (
132129
SELECT
133130
MS.project_id,
134131
MS.group_id,
135132
MS.user_id,
136133
MSUR.user_group_id,
137-
MAX(MS.start_time::date) as timestamp_date,
138-
MIN(MS.start_time) as start_time,
139-
MAX(MS.end_time) as end_time,
140-
COUNT(DISTINCT T.task_id) as task_count,
141-
SUM(T.area) as area_swiped
134+
MS.start_time::date as timestamp_date,
135+
MS.start_time as start_time,
136+
MS.end_time as end_time,
137+
MS.items_count as task_count,
138+
TG.total_task_group_area as area_swiped
142139
From mapping_sessions_user_groups MSUR
143140
INNER JOIN mapping_sessions MS USING (mapping_session_id)
144-
INNER JOIN mapping_sessions_results MSR USING (mapping_session_id)
145-
INNER JOIN task_data T USING (task_id)
141+
INNER JOIN task_group_data TG USING (project_id, group_id)
146142
WHERE
147143
MS.start_time >= %(from_date)s and MS.start_time < %(until_date)s
148-
GROUP BY MS.project_id, MS.group_id, MS.user_id, MSUR.user_group_id
149144
),
150-
-- Aggregate group data
145+
-- Additional aggregate by timestamp_date
151146
user_group_agg_data as (
152147
SELECT
153148
project_id,

‎django/apps/existing_database/test_queries.py‎

Lines changed: 16 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,11 @@ def setUpTestData(cls):
2727
cls.user = UserFactory.create()
2828
# Project, Group, Task
2929
cls.project = ProjectFactory.create()
30-
cls.groups = GroupFactory.create_batch(3, project=cls.project)
30+
cls.groups = GroupFactory.create_batch(4, project=cls.project)
3131
cls.tasks = [
3232
task
3333
for group in cls.groups
34-
for task in TaskFactory.create_batch(3, group=group)
34+
for task in TaskFactory.create_batch(4, group=group)
3535
]
3636
cls.mapping_sessions = {
3737
group.group_id: MappingSessionFactory(
@@ -40,6 +40,7 @@ def setUpTestData(cls):
4040
user=cls.user,
4141
start_time=now,
4242
end_time=now,
43+
items_count=2,
4344
)
4445
for group in cls.groups
4546
}
@@ -53,8 +54,8 @@ def setUpTestData(cls):
5354
for task in tasks
5455
]
5556
for value, tasks in [
56-
(0, cls.tasks[:5]),
57-
(1, cls.tasks[5:]),
57+
(0, cls.tasks[:8]),
58+
(1, cls.tasks[8:]),
5859
]
5960
}
6061
cls.user_groups = UserGroupFactory.create_batch(4)
@@ -89,19 +90,16 @@ def test_community_stats(self):
8990
"""
9091

9192
resp = self.query_check(query)
92-
self.assertEqual(
93-
resp["data"],
94-
dict(
95-
communityStats=dict(
96-
totalContributors=1,
97-
totalSwipes=9,
98-
totalUserGroups=3,
99-
),
100-
communityStatsLatest=dict(
101-
totalContributors=1,
102-
totalSwipes=9,
103-
totalUserGroups=3,
104-
),
93+
assert resp["data"] == dict(
94+
communityStats=dict(
95+
totalContributors=1,
96+
totalSwipes=8,
97+
totalUserGroups=3,
98+
),
99+
communityStatsLatest=dict(
100+
totalContributors=1,
101+
totalSwipes=8,
102+
totalUserGroups=3,
105103
),
106104
)
107105

@@ -126,7 +124,7 @@ def test_user_group_aggregated_calc(self):
126124
totalMappingProjects=1,
127125
totalOrganization=0,
128126
totalSwipeTime=0,
129-
totalSwipes=6,
127+
totalSwipes=16,
130128
)
131129
without_data = dict(
132130
totalAreaSwiped=0,

0 commit comments

Comments
 (0)