Skip to content

Commit 1ea2291

Browse files
ElJochoElJocho
committed
feat: moved deletion of results into project_types to account for differentiating result tables
1 parent 3a393b5 commit 1ea2291

File tree

4 files changed

+154
-55
lines changed

4 files changed

+154
-55
lines changed

‎mapswipe_workers/mapswipe_workers/firebase_to_postgres/delete_project.py‎

Lines changed: 12 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
from firebase_admin import exceptions
99

1010
from mapswipe_workers import auth
11-
from mapswipe_workers.definitions import CustomError, logger
11+
from mapswipe_workers.definitions import CustomError, logger, ProjectType
1212

1313

1414
def chunks(data: list, size: int = 250) -> Iterable[list]:
@@ -94,48 +94,18 @@ def delete_project(project_ids: list) -> bool:
9494
ref.delete()
9595

9696
pg_db = auth.postgresDB()
97-
sql_query = """
98-
DELETE FROM mapping_sessions_results msr
99-
USING mapping_sessions ms
100-
WHERE ms.mapping_session_id = msr.mapping_session_id
101-
AND ms.project_id = %(project_id)s;
102-
"""
103-
pg_db.query(sql_query, {"project_id": project_id})
97+
try:
98+
project_type = pg_db.retr_query(
99+
f"Select project_type from projects where project_id = %(project_id)s;",
100+
{"project_id": project_id}
101+
)[0][0]
102+
project = ProjectType(project_type).constructor
103+
project.delete_from_postgres(project_id)
104104

105-
pg_db = auth.postgresDB()
106-
sql_query = """
107-
DELETE FROM mapping_sessions_results_geometry msr
108-
USING mapping_sessions ms
109-
WHERE ms.mapping_session_id = msr.mapping_session_id
110-
AND ms.project_id = %(project_id)s;
111-
"""
112-
pg_db.query(sql_query, {"project_id": project_id})
105+
except IndexError:
106+
logger.info(
107+
f"Tried to delete project which does not exist in postgres: {project_id}"
108+
)
113109

114-
sql_query = """
115-
DELETE
116-
FROM mapping_sessions
117-
WHERE project_id = %(project_id)s ;
118-
"""
119-
pg_db.query(sql_query, {"project_id": project_id})
120-
sql_query = "DELETE FROM tasks WHERE project_id = %(project_id)s;"
121-
pg_db.query(sql_query, {"project_id": project_id})
122-
sql_query = "DELETE FROM groups WHERE project_id = %(project_id)s;"
123-
pg_db.query(sql_query, {"project_id": project_id})
124-
# -- Table from django/apps/aggregated/models.py. Used to cache stats data
125-
# NOTE: Django doesn't support database-level CASCADE delete
126-
# https://docs.djangoproject.com/en/4.1/ref/models/fields/#django.db.models.ForeignKey.on_delete
127-
for aggregated_table_name in [
128-
"aggregated_aggregateduserstatdata",
129-
"aggregated_aggregatedusergroupstatdata",
130-
]:
131-
if pg_db.table_exists(aggregated_table_name):
132-
sql_query = f"""
133-
DELETE FROM {aggregated_table_name}
134-
WHERE project_id = %(project_id)s;
135-
"""
136-
pg_db.query(sql_query, {"project_id": project_id})
137-
# Finally delete the project
138-
sql_query = "DELETE FROM projects WHERE project_id = %(project_id)s;"
139-
pg_db.query(sql_query, {"project_id": project_id})
140110

141111
return True

‎mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/digitization/project.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
from urllib.request import urlretrieve
22

3+
from overrides import overrides
4+
5+
from mapswipe_workers import auth
36
from mapswipe_workers.firebase.firebase import Firebase
47
from mapswipe_workers.firebase_to_postgres.transfer_results import (
58
results_to_file,
@@ -55,3 +58,15 @@ def results_to_postgres(results: dict, project_id: str, filter_mode: bool):
5558
def get_per_project_statistics(project_id, project_info):
5659
"""How to aggregate the project results."""
5760
return get_statistics_for_geometry_result_project(project_id)
61+
62+
@staticmethod
63+
@overrides()
64+
def delete_mapping_session_results(project_id):
65+
p_con = auth.postgresDB()
66+
sql_query = """
67+
DELETE FROM mapping_sessions_results_geometry msr
68+
USING mapping_sessions ms
69+
WHERE ms.mapping_session_id = msr.mapping_session_id
70+
AND ms.project_id = %(project_id)s;
71+
"""
72+
p_con.query(sql_query, {"project_id": project_id})

‎mapswipe_workers/mapswipe_workers/project_types/project.py‎

Lines changed: 40 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ def save_project(self):
143143
f"{self.projectId}" f" - the project has been saved" f" to files"
144144
)
145145
except Exception as e:
146-
self.delete_from_postgres()
146+
self.delete_from_postgres(self.projectId)
147147
logger.exception(
148148
f"{self.projectId}" f" - the project could not be saved" f" to files. "
149149
)
@@ -165,7 +165,7 @@ def save_project(self):
165165
# if project can't be saved to firebase, delete also in postgres
166166
except Exception as e:
167167
self.delete_draft_from_firebase()
168-
self.delete_from_postgres()
168+
self.delete_from_postgres(self.projectId)
169169
self.delete_from_files()
170170
logger.exception(
171171
f"{self.projectId}"
@@ -562,23 +562,50 @@ def delete_draft_from_firebase(self):
562562
firebase = Firebase()
563563
firebase.delete_project_draft_from_firebase(self.projectId)
564564

565-
def delete_from_postgres(self):
565+
@staticmethod
566+
def delete_mapping_session_results(project_id):
567+
p_con = auth.postgresDB()
568+
sql_query = """
569+
DELETE FROM mapping_sessions_results msr
570+
USING mapping_sessions ms
571+
WHERE ms.mapping_session_id = msr.mapping_session_id
572+
AND ms.project_id = %(project_id)s;
573+
"""
574+
p_con.query(sql_query, {"project_id": project_id})
575+
576+
@classmethod
577+
def delete_from_postgres(cls, project_id):
566578
p_con = auth.postgresDB()
567579

580+
cls.delete_mapping_session_results(project_id)
581+
568582
sql_query = """
569-
DELETE FROM tasks WHERE project_id = %s;
570-
DELETE FROM groups WHERE project_id = %s;
571-
DELETE FROM projects WHERE project_id = %s;
583+
DELETE FROM mapping_sessions WHERE project_id = %(project_id)s;
584+
DELETE FROM tasks WHERE project_id = %(project_id)s;
585+
DELETE FROM groups WHERE project_id = %(project_id)s;
572586
"""
573-
data = [
574-
self.projectId,
575-
self.projectId,
576-
self.projectId,
577-
]
578-
p_con.query(sql_query, data)
587+
p_con.query(sql_query, {"project_id": project_id})
588+
589+
# -- Table from django/apps/aggregated/models.py. Used to cache stats data
590+
# NOTE: Django doesn't support database-level CASCADE delete
591+
# https://docs.djangoproject.com/en/4.1/ref/models/fields/#django.db.models.ForeignKey.on_delete
592+
for aggregated_table_name in [
593+
"aggregated_aggregateduserstatdata",
594+
"aggregated_aggregatedusergroupstatdata",
595+
]:
596+
if p_con.table_exists(aggregated_table_name):
597+
sql_query = f"""
598+
DELETE FROM {aggregated_table_name}
599+
WHERE project_id = %(project_id)s;
600+
"""
601+
p_con.query(sql_query, {"project_id": project_id})
602+
603+
sql_query = """DELETE FROM projects WHERE project_id = %(project_id)s;"""
604+
p_con.query(sql_query, {"project_id": project_id})
605+
579606
del p_con
580607
logger.info(
581-
f"{self.projectId} - "
608+
f"{project_id} - "
582609
f"deleted project, groups and tasks "
583610
f"from postgres"
584611
)

‎mapswipe_workers/tests/integration/test_delete_project_digitization.py‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
import time
2+
import unittest
3+
4+
from mapswipe_workers import auth
5+
from mapswipe_workers.config import FIREBASE_DB
6+
from mapswipe_workers.definitions import CustomError
7+
from mapswipe_workers.firebase_to_postgres import delete_project
8+
from tests.integration import base, set_up, tear_down
9+
10+
11+
class TestDeleteProjectDigitization(base.BaseTestCase):
12+
def setUp(self):
13+
self.project_id = project_type = fixture_name = "digitization"
14+
15+
set_up.create_test_project(
16+
project_type,
17+
fixture_name,
18+
results=True,
19+
mapping_sessions_results="mapping_sessions_results_geometry",
20+
)
21+
22+
def tearDown(self):
23+
tear_down.delete_test_data(self.project_id)
24+
25+
26+
def verify_postgres_not_empty(self):
27+
pg_db = auth.postgresDB()
28+
sql_query = f"SELECT * FROM tasks WHERE project_id = '{self.project_id}'"
29+
result = pg_db.retr_query(sql_query)
30+
self.assertGreater(len(result), 0)
31+
sql_query = f"SELECT * FROM groups WHERE project_id = '{self.project_id}'"
32+
result = pg_db.retr_query(sql_query)
33+
self.assertGreater(len(result), 0)
34+
sql_query = f"SELECT * FROM projects WHERE project_id = '{self.project_id}'"
35+
result = pg_db.retr_query(sql_query)
36+
self.assertGreater(len(result), 0)
37+
sql_query = (
38+
f"SELECT * FROM mapping_sessions WHERE project_id = '{self.project_id}'"
39+
)
40+
result = pg_db.retr_query(sql_query)
41+
self.assertGreater(len(result), 0)
42+
sql_query = f"""
43+
SELECT msr.*
44+
FROM mapping_sessions_results_geometry msr
45+
JOIN mapping_sessions ms USING (mapping_session_id)
46+
WHERE ms.project_id = '{self.project_id}'
47+
"""
48+
result = pg_db.retr_query(sql_query)
49+
self.assertGreater(len(result), 0)
50+
51+
def verify_postgres_empty(self):
52+
pg_db = auth.postgresDB()
53+
sql_query = f"SELECT * FROM tasks WHERE project_id = '{self.project_id}'"
54+
result = pg_db.retr_query(sql_query)
55+
self.assertListEqual(result, [])
56+
sql_query = f"SELECT * FROM groups WHERE project_id = '{self.project_id}'"
57+
result = pg_db.retr_query(sql_query)
58+
self.assertListEqual(result, [])
59+
sql_query = f"SELECT * FROM projects WHERE project_id = '{self.project_id}'"
60+
result = pg_db.retr_query(sql_query)
61+
self.assertListEqual(result, [])
62+
sql_query = (
63+
f"SELECT * FROM mapping_sessions WHERE project_id = '{self.project_id}'"
64+
)
65+
result = pg_db.retr_query(sql_query)
66+
self.assertListEqual(result, [])
67+
sql_query = f"""
68+
SELECT msr.*
69+
FROM mapping_sessions_results_geometry msr
70+
JOIN mapping_sessions ms USING (mapping_session_id)
71+
WHERE ms.project_id = '{self.project_id}'
72+
"""
73+
74+
result = pg_db.retr_query(sql_query)
75+
self.assertListEqual(result, [])
76+
77+
def test_deletion(self):
78+
"""Test if tasks, groups, project and results are deleted."""
79+
self.verify_postgres_not_empty()
80+
delete_project.delete_project([self.project_id])
81+
time.sleep(1) # Wait for Firebase Functions to complete
82+
83+
self.verify_postgres_empty()
84+
85+
86+
if __name__ == "__main__":
87+
unittest.main()

0 commit comments

Comments
 (0)