Skip to content

Commit 1c5f7f4

Browse files
Matthias-SchaubMatthias-Schaub
authored
Merge pull request #320 from mapswipe/delete-project
Delete project
2 parents 461f13e + 6a82e7b commit 1c5f7f4

File tree

4 files changed

+299
-36
lines changed

4 files changed

+299
-36
lines changed

‎mapswipe_workers/mapswipe_workers/firebase_to_postgres/archive_project.py‎

Lines changed: 35 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,20 @@
22
Archive a project.
33
"""
44

5+
from typing import Iterable
6+
7+
from firebase_admin import exceptions
8+
59
from mapswipe_workers import auth
610
from mapswipe_workers.definitions import logger
711

812

13+
def chunks(data: list, size: int = 250) -> Iterable[list]:
14+
"""Yield successive n-sized chunks from list."""
15+
for i in range(0, len(data), size):
16+
yield data[i : i + size]
17+
18+
919
def archive_project(project_ids: list) -> None:
1020
"""
1121
Archive a project.
@@ -15,46 +25,38 @@ def archive_project(project_ids: list) -> None:
1525
"""
1626
for project_id in project_ids:
1727
logger.info(f"Archive project with the id {project_id}")
18-
logger.info(f"Delete results of project with the id {project_id}")
1928

2029
fb_db = auth.firebaseDB()
21-
fb_db.reference(f"v2/results/{project_id}").set({})
22-
23-
# get group keys for this project to estimate size in firebase
24-
groups = fb_db.reference(f"v2/groups/{project_id}").get(shallow=True)
25-
26-
if not groups:
27-
logger.info("no groups to delete in firebase")
28-
else:
29-
group_keys = list(groups.keys())
30-
chunk_size = 250
31-
chunks = int(len(group_keys) / chunk_size) + 1
32-
33-
# delete groups, tasks in firebase for each chunk using the update function
34-
for i in range(0, chunks):
35-
logger.info(
36-
f"Delete max {chunk_size} groups and tasks"
37-
f"of project with the id {project_id}"
38-
)
39-
update_dict = {}
40-
for group_id in group_keys[:chunk_size]:
41-
update_dict[group_id] = None
42-
fb_db.reference(f"v2/groups/{project_id}").update(update_dict)
43-
fb_db.reference(f"v2/tasks/{project_id}").update(update_dict)
44-
group_keys = group_keys[chunk_size:]
45-
46-
logger.info(
47-
f"Set status=archived in Firebase for project with the id {project_id}"
48-
)
49-
fb_db = auth.firebaseDB()
30+
ref = fb_db.reference(f"v2/results/{project_id}")
31+
try:
32+
ref.delete()
33+
except exceptions.InvalidArgumentError:
34+
# Data to write exceeds the maximum size that can be modified
35+
# with a single request. Delete chunks of data instead.
36+
childs = ref.get(shallow=True)
37+
for chunk in chunks(list(childs.keys())):
38+
ref.update({key: None for key in chunk})
39+
ref.delete()
40+
41+
ref = fb_db.reference(f"v2/results/{project_id}")
42+
try:
43+
ref.delete()
44+
except exceptions.InvalidArgumentError:
45+
# Data to write exceeds the maximum size that can be modified
46+
# with a single request. Delete chunks of data instead.
47+
childs = ref.get(shallow=True)
48+
for chunk in chunks(list(childs.keys())):
49+
ref.update({key: None for key in chunk})
50+
ref.delete()
51+
52+
fb_db.reference(f"v2/groups/{project_id}").delete()
5053
fb_db.reference(f"v2/projects/{project_id}/status").set("archived")
5154

52-
logger.info(
53-
f"Set status=archived in Postgres for project with the id {project_id}"
54-
)
5555
pg_db = auth.postgresDB()
5656
sql_query = """
5757
UPDATE projects SET status = 'archived'
5858
WHERE project_id = %(project_id)s;
5959
"""
6060
pg_db.query(sql_query, {"project_id": project_id})
61+
62+
return True

‎mapswipe_workers/mapswipe_workers/firebase_to_postgres/delete_project.py‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
"""
2+
Delete projects.
3+
"""
4+
5+
from typing import Iterable
6+
7+
from firebase_admin import exceptions
8+
9+
from mapswipe_workers import auth
10+
from mapswipe_workers.definitions import logger
11+
12+
13+
def chunks(data: list, size: int = 250) -> Iterable[list]:
14+
"""Yield successive n-sized chunks from list."""
15+
for i in range(0, len(data), size):
16+
yield data[i : i + size]
17+
18+
19+
def delete_project(project_ids: list) -> None:
20+
"""
21+
Deletes project, groups, tasks and results from Firebase and Postgres.
22+
"""
23+
for project_id in project_ids:
24+
logger.info(
25+
f"Delete project, groups, tasks and results of project: {project_id}"
26+
)
27+
28+
fb_db = auth.firebaseDB()
29+
30+
ref = fb_db.reference(f"v2/results/{project_id}")
31+
try:
32+
ref.delete()
33+
except exceptions.InvalidArgumentError:
34+
# Data to write exceeds the maximum size that can be modified
35+
# with a single request. Delete chunks of data instead.
36+
childs = ref.get(shallow=True)
37+
for chunk in chunks(list(childs.keys())):
38+
ref.update({key: None for key in chunk})
39+
ref.delete()
40+
41+
ref = fb_db.reference(f"v2/tasks/{project_id}")
42+
try:
43+
ref.delete()
44+
except exceptions.InvalidArgumentError:
45+
# Data to write exceeds the maximum size that can be modified
46+
# with a single request. Delete chunks of data instead.
47+
childs = ref.get(shallow=True)
48+
for chunk in chunks(list(childs.keys())):
49+
ref.update({key: None for key in chunk})
50+
ref.delete()
51+
52+
fb_db.reference(f"v2/groups/{project_id}").delete()
53+
fb_db.reference(f"v2/projects/{project_id}").delete()
54+
55+
pg_db = auth.postgresDB()
56+
sql_query = "DELETE FROM results WHERE project_id = {};".format(project_id)
57+
pg_db.query(sql_query, project_id)
58+
sql_query = "DELETE FROM tasks WHERE project_id = {};".format(project_id)
59+
pg_db.query(sql_query, project_id)
60+
sql_query = "DELETE FROM groups WHERE project_id = {};".format(project_id)
61+
pg_db.query(sql_query, project_id)
62+
sql_query = "DELETE FROM projects WHERE project_id = {};".format(project_id)
63+
pg_db.query(sql_query, project_id)
64+
65+
return True

‎mapswipe_workers/mapswipe_workers/mapswipe_workers.py‎

Lines changed: 53 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,14 @@
44
import json
55
import time
66

7-
import click
87
import schedule as sched
8+
9+
import click
910
from mapswipe_workers import auth
1011
from mapswipe_workers.definitions import CustomError, logger, sentry
1112
from mapswipe_workers.firebase_to_postgres import (
1213
archive_project,
14+
delete_project,
1315
transfer_results,
1416
update_data,
1517
)
@@ -198,11 +200,59 @@ def run_create_tutorial(input_file) -> None:
198200
)
199201
def run_archive_project(project_id, project_ids):
200202
"""Archive projects in Postgres. Delete groups, tasks and results from Firebase."""
201-
if not project_ids:
203+
if not project_ids and not project_id:
204+
click.echo("Missing argument")
205+
return None
206+
elif not project_ids:
202207
project_ids = [project_id]
208+
click.echo("Start archive")
203209
update_data.update_project_data(project_ids)
204210
transfer_results.transfer_results(project_ids)
205-
archive_project.archive_project(project_ids)
211+
if archive_project.archive_project(project_ids):
212+
click.echo("Finished archive")
213+
214+
215+
@cli.command("delete")
216+
@click.option(
217+
"--project-id", "-i", help=("Delete project with giving project id"), type=str,
218+
)
219+
@click.option(
220+
"--project-ids",
221+
cls=PythonLiteralOption,
222+
default="[]",
223+
help=(
224+
f"Delete multiple projects. "
225+
f"Provide project id strings as a list: "
226+
f"""["project_a", "project_b"]"""
227+
),
228+
)
229+
def run_delete_project(project_id, project_ids):
230+
"""Delete tasks, groups, project and results."""
231+
if not project_ids and not project_id:
232+
click.echo("Missing argument")
233+
return None
234+
elif not project_ids:
235+
project_ids = [project_id]
236+
237+
click.echo(
238+
"Projects and all associated data including results "
239+
+ "with following project ids will be deleted permantly:"
240+
)
241+
for project_id in project_ids:
242+
click.echo(project_id)
243+
click.echo()
244+
click.echo("Continue with deletion? [y/n] ", nl=False)
245+
click.echo()
246+
c = click.getchar()
247+
248+
if c == "y":
249+
click.echo("Start deletion")
250+
if delete_project.delete_project(project_ids):
251+
click.echo("Finished deletions")
252+
elif c == "n":
253+
click.echo("Abort!")
254+
else:
255+
click.echo("Invalid input")
206256

207257

208258
@cli.command("run")

‎mapswipe_workers/tests/test_delete_project.py‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
import unittest
2+
3+
import set_up
4+
import tear_down
5+
from mapswipe_workers import auth
6+
from mapswipe_workers.firebase_to_postgres import delete_project
7+
8+
9+
class TestDeleteProject(unittest.TestCase):
10+
def setUp(self):
11+
self.project_id = set_up.create_test_project("build_area")
12+
13+
def tearDown(self):
14+
tear_down.delete_test_project(self.project_id)
15+
16+
def test_deletion(self):
17+
"""Test if tasks, groups, project and results are deleted."""
18+
delete_project.delete_project([self.project_id])
19+
20+
fb_db = auth.firebaseDB()
21+
ref = fb_db.reference("v2/results/{0}".format(self.project_id))
22+
self.assertIsNone(ref.get())
23+
ref = fb_db.reference("v2/tasks/{0}".format(self.project_id))
24+
self.assertIsNone(ref.get())
25+
ref = fb_db.reference("v2/groups/{0}".format(self.project_id))
26+
self.assertIsNone(ref.get())
27+
ref = fb_db.reference("v2/projects/{0}".format(self.project_id))
28+
self.assertIsNone(ref.get())
29+
30+
pg_db = auth.postgresDB()
31+
sql_query = "SELECT * FROM tasks WHERE project_id = {}".format(self.project_id)
32+
result = pg_db.retr_query(sql_query)
33+
self.assertIsNone(result)
34+
sql_query = "SELECT * FROM groups WHERE project_id = {}".format(self.project_id)
35+
result = pg_db.retr_query(sql_query)
36+
self.assertIsNone(result)
37+
sql_query = "SELECT * FROM projects WHERE project_id = {}".format(
38+
self.project_id
39+
)
40+
result = pg_db.retr_query(sql_query)
41+
self.assertIsNone(result)
42+
sql_query = "SELECT * FROM results WHERE project_id = {}".format(
43+
self.project_id
44+
)
45+
result = pg_db.retr_query(sql_query)
46+
self.assertIsNone(result)
47+
48+
def test_project_id_equals_none(self):
49+
"""Test for project id equals None."""
50+
delete_project.delete_project([None])
51+
52+
fb_db = auth.firebaseDB()
53+
ref = fb_db.reference("v2/results/{0}".format(self.project_id))
54+
self.assertIsNotNone(ref.get())
55+
ref = fb_db.reference("v2/tasks/{0}".format(self.project_id))
56+
self.assertIsNotNone(ref.get())
57+
ref = fb_db.reference("v2/groups/{0}".format(self.project_id))
58+
self.assertIsNotNone(ref.get())
59+
ref = fb_db.reference("v2/projects/{0}".format(self.project_id))
60+
self.assertIsNotNone(ref.get())
61+
62+
pg_db = auth.postgresDB()
63+
sql_query = "SELECT * FROM tasks WHERE project_id = {}".format(self.project_id)
64+
result = pg_db.retr_query(sql_query)
65+
self.assertIsNotNone(result)
66+
sql_query = "SELECT * FROM groups WHERE project_id = {}".format(self.project_id)
67+
result = pg_db.retr_query(sql_query)
68+
self.assertIsNotNone(result)
69+
sql_query = "SELECT * FROM projects WHERE project_id = {}".format(
70+
self.project_id
71+
)
72+
result = pg_db.retr_query(sql_query)
73+
self.assertIsNotNone(result)
74+
sql_query = "SELECT * FROM results WHERE project_id = {}".format(
75+
self.project_id
76+
)
77+
result = pg_db.retr_query(sql_query)
78+
self.assertIsNotNone(result)
79+
80+
def test_project_id_equals_empty_str(self):
81+
"""Test for poject id equals empty string."""
82+
delete_project.delete_project([""])
83+
84+
fb_db = auth.firebaseDB()
85+
ref = fb_db.reference("v2/results/{0}".format(self.project_id))
86+
self.assertIsNotNone(ref.get())
87+
ref = fb_db.reference("v2/tasks/{0}".format(self.project_id))
88+
self.assertIsNotNone(ref.get())
89+
ref = fb_db.reference("v2/groups/{0}".format(self.project_id))
90+
self.assertIsNotNone(ref.get())
91+
ref = fb_db.reference("v2/projects/{0}".format(self.project_id))
92+
self.assertIsNotNone(ref.get())
93+
94+
pg_db = auth.postgresDB()
95+
sql_query = "SELECT * FROM tasks WHERE project_id = {}".format(self.project_id)
96+
result = pg_db.retr_query(sql_query)
97+
self.assertIsNotNone(result)
98+
sql_query = "SELECT * FROM groups WHERE project_id = {}".format(self.project_id)
99+
result = pg_db.retr_query(sql_query)
100+
self.assertIsNotNone(result)
101+
sql_query = "SELECT * FROM projects WHERE project_id = {}".format(
102+
self.project_id
103+
)
104+
result = pg_db.retr_query(sql_query)
105+
self.assertIsNotNone(result)
106+
sql_query = "SELECT * FROM results WHERE project_id = {}".format(
107+
self.project_id
108+
)
109+
result = pg_db.retr_query(sql_query)
110+
self.assertIsNotNone(result)
111+
112+
def test_project_id_not_exists(self):
113+
"""Test for project id which does not exists."""
114+
delete_project.delete_project(["tuna"])
115+
116+
fb_db = auth.firebaseDB()
117+
ref = fb_db.reference("v2/results/{0}".format(self.project_id))
118+
self.assertIsNotNone(ref.get())
119+
ref = fb_db.reference("v2/tasks/{0}".format(self.project_id))
120+
self.assertIsNotNone(ref.get())
121+
ref = fb_db.reference("v2/groups/{0}".format(self.project_id))
122+
self.assertIsNotNone(ref.get())
123+
ref = fb_db.reference("v2/projects/{0}".format(self.project_id))
124+
self.assertIsNotNone(ref.get())
125+
126+
pg_db = auth.postgresDB()
127+
sql_query = "SELECT * FROM tasks WHERE project_id = {}".format(self.project_id)
128+
result = pg_db.retr_query(sql_query)
129+
self.assertIsNotNone(result)
130+
sql_query = "SELECT * FROM groups WHERE project_id = {}".format(self.project_id)
131+
result = pg_db.retr_query(sql_query)
132+
self.assertIsNotNone(result)
133+
sql_query = "SELECT * FROM projects WHERE project_id = {}".format(
134+
self.project_id
135+
)
136+
result = pg_db.retr_query(sql_query)
137+
self.assertIsNotNone(result)
138+
sql_query = "SELECT * FROM results WHERE project_id = {}".format(
139+
self.project_id
140+
)
141+
result = pg_db.retr_query(sql_query)
142+
self.assertIsNotNone(result)
143+
144+
145+
if __name__ == "__main__":
146+
unittest.main()

0 commit comments

Comments
 (0)