Skip to content
This repository was archived by the owner on May 5, 2025. It is now read-only.

Commit b6fd340

Browse files
committed
add FlareCleanupTask
1 parent 0fd7a81 commit b6fd340

File tree

3 files changed

+198
-0
lines changed

3 files changed

+198
-0
lines changed

celery_config.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
from shared.celery_config import (
1111
BaseCeleryConfig,
1212
brolly_stats_rollup_task_name,
13+
flare_cleanup_task_name,
1314
gh_app_webhook_check_task_name,
1415
health_check_task_name,
1516
profiling_finding_task_name,
@@ -94,6 +95,13 @@ def _beat_schedule():
9495
"cron_task_generation_time_iso": BeatLazyFunc(get_utc_now_as_iso_format)
9596
},
9697
},
98+
"flare_cleanup": {
99+
"task": flare_cleanup_task_name,
100+
"schedule": crontab(minute="0", hour="4"), # every day, 4am UTC (8pm PT)
101+
"kwargs": {
102+
"cron_task_generation_time_iso": BeatLazyFunc(get_utc_now_as_iso_format)
103+
},
104+
},
97105
}
98106

99107
if get_config("setup", "find_uncollected_profilings", "enabled", default=True):

tasks/flare_cleanup.py

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
import logging
2+
3+
from shared.api_archive.archive import ArchiveService
4+
from shared.celery_config import flare_cleanup_task_name
5+
from shared.django_apps.core.models import Pull, PullStates
6+
7+
from app import celery_app
8+
from tasks.crontasks import CodecovCronTask
9+
10+
log = logging.getLogger(__name__)
11+
12+
13+
class FlareCleanupTask(CodecovCronTask, name=flare_cleanup_task_name):
14+
"""
15+
Flare is a field on a Pull object.
16+
Flare is used to draw static graphs (see GraphHandler view in api) and can be large.
17+
The majority of flare graphs are used in pr comments, so we keep the (maybe large) flare "available"
18+
in either the db or Archive storage while the pull is OPEN.
19+
If the pull is not OPEN, we dump the flare to save space.
20+
If we need to generate a flare graph for a non-OPEN pull, we build_report_from_commit
21+
and generate fresh flare from that report (see GraphHandler view in api).
22+
"""
23+
24+
@classmethod
25+
def get_min_seconds_interval_between_executions(cls):
26+
return 72000 # 20h
27+
28+
def run_cron_task(self, db_session, *args, **kwargs):
29+
# for any Pull that is not OPEN, clear the flare field(s)
30+
non_open_pulls = Pull.objects.exclude(state=PullStates.OPEN.value)
31+
32+
log.info("Starting FlareCleanupTask")
33+
34+
# clear in db
35+
non_open_pulls_with_flare_in_db = non_open_pulls.filter(
36+
_flare__isnull=False
37+
).exclude(_flare={})
38+
# single query, objs are not loaded into memory, does not call .save(), does not refresh updatestamp
39+
n_updated = non_open_pulls_with_flare_in_db.update(_flare=None)
40+
log.info(f"FlareCleanupTask cleared {n_updated} _flares")
41+
42+
# clear in Archive
43+
non_open_pulls_with_flare_in_archive = non_open_pulls.filter(
44+
_flare_storage_path__isnull=False
45+
).select_related("repository")
46+
log.info(
47+
f"FlareCleanupTask will clear {non_open_pulls_with_flare_in_archive.count()} Archive flares"
48+
)
49+
# single query, loads all pulls and repos in qset into memory, deletes file in Archive 1 by 1
50+
for pull in non_open_pulls_with_flare_in_archive:
51+
archive_service = ArchiveService(repository=pull.repository)
52+
archive_service.delete_file(pull._flare_storage_path)
53+
54+
# single query, objs are not loaded into memory, does not call .save(), does not refresh updatestamp
55+
n_updated = non_open_pulls_with_flare_in_archive.update(
56+
_flare_storage_path=None
57+
)
58+
59+
log.info(f"FlareCleanupTask cleared {n_updated} Archive flares")
60+
61+
62+
RegisteredFlareCleanupTask = celery_app.register_task(FlareCleanupTask())
63+
flare_cleanup_task = celery_app.tasks[RegisteredFlareCleanupTask.name]
Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
import json
2+
from unittest.mock import call
3+
4+
from shared.django_apps.core.models import Pull, PullStates
5+
from shared.django_apps.core.tests.factories import PullFactory, RepositoryFactory
6+
7+
from tasks.flare_cleanup import FlareCleanupTask
8+
9+
10+
class TestFlareCleanupTask(object):
11+
def test_get_min_seconds_interval_between_executions(self):
12+
assert isinstance(
13+
FlareCleanupTask.get_min_seconds_interval_between_executions(),
14+
int,
15+
)
16+
assert FlareCleanupTask.get_min_seconds_interval_between_executions() > 17000
17+
18+
def test_successful_run(self, transactional_db, mocker):
19+
mock_logs = mocker.patch("logging.Logger.info")
20+
mock_archive_service = mocker.patch(
21+
"shared.django_apps.utils.model_utils.ArchiveService"
22+
)
23+
archive_value_for_flare = {"some": "data"}
24+
mock_archive_service.return_value.read_file.return_value = json.dumps(
25+
archive_value_for_flare
26+
)
27+
mock_path = "path/to/written/object"
28+
mock_archive_service.return_value.write_json_data_to_storage.return_value = (
29+
mock_path
30+
)
31+
mock_archive_service_in_task = mocker.patch(
32+
"tasks.flare_cleanup.ArchiveService"
33+
)
34+
mock_archive_service_in_task.return_value.delete_file.return_value = None
35+
36+
local_value_for_flare = {"test": "test"}
37+
open_pull_with_local_flare = PullFactory(
38+
state=PullStates.OPEN.value,
39+
_flare=local_value_for_flare,
40+
repository=RepositoryFactory(),
41+
)
42+
assert open_pull_with_local_flare.flare == local_value_for_flare
43+
assert open_pull_with_local_flare._flare == local_value_for_flare
44+
assert open_pull_with_local_flare._flare_storage_path is None
45+
46+
closed_pull_with_local_flare = PullFactory(
47+
state=PullStates.CLOSED.value,
48+
_flare=local_value_for_flare,
49+
repository=RepositoryFactory(),
50+
)
51+
assert closed_pull_with_local_flare.flare == local_value_for_flare
52+
assert closed_pull_with_local_flare._flare == local_value_for_flare
53+
assert closed_pull_with_local_flare._flare_storage_path is None
54+
55+
open_pull_with_archive_flare = PullFactory(
56+
state=PullStates.OPEN.value,
57+
_flare=None,
58+
_flare_storage_path=mock_path,
59+
repository=RepositoryFactory(),
60+
)
61+
assert open_pull_with_archive_flare.flare == archive_value_for_flare
62+
assert open_pull_with_archive_flare._flare is None
63+
assert open_pull_with_archive_flare._flare_storage_path == mock_path
64+
65+
merged_pull_with_archive_flare = PullFactory(
66+
state=PullStates.MERGED.value,
67+
_flare=None,
68+
_flare_storage_path=mock_path,
69+
repository=RepositoryFactory(),
70+
)
71+
assert merged_pull_with_archive_flare.flare == archive_value_for_flare
72+
assert merged_pull_with_archive_flare._flare is None
73+
assert merged_pull_with_archive_flare._flare_storage_path == mock_path
74+
75+
task = FlareCleanupTask()
76+
task.run_cron_task(transactional_db)
77+
78+
mock_logs.assert_has_calls(
79+
[
80+
call("Starting FlareCleanupTask"),
81+
call("FlareCleanupTask cleared 1 _flares"),
82+
call("FlareCleanupTask will clear 1 Archive flares"),
83+
call("FlareCleanupTask cleared 1 Archive flares"),
84+
]
85+
)
86+
87+
# there is a cache for flare on the object (all ArchiveFields have this),
88+
# so get a fresh copy of each object without the cached value
89+
open_pull_with_local_flare = Pull.objects.get(id=open_pull_with_local_flare.id)
90+
assert open_pull_with_local_flare.flare == local_value_for_flare
91+
assert open_pull_with_local_flare._flare == local_value_for_flare
92+
assert open_pull_with_local_flare._flare_storage_path is None
93+
94+
closed_pull_with_local_flare = Pull.objects.get(
95+
id=closed_pull_with_local_flare.id
96+
)
97+
assert closed_pull_with_local_flare.flare == {}
98+
assert closed_pull_with_local_flare._flare is None
99+
assert closed_pull_with_local_flare._flare_storage_path is None
100+
101+
open_pull_with_archive_flare = Pull.objects.get(
102+
id=open_pull_with_archive_flare.id
103+
)
104+
assert open_pull_with_archive_flare.flare == archive_value_for_flare
105+
assert open_pull_with_archive_flare._flare is None
106+
assert open_pull_with_archive_flare._flare_storage_path == mock_path
107+
108+
merged_pull_with_archive_flare = Pull.objects.get(
109+
id=merged_pull_with_archive_flare.id
110+
)
111+
assert merged_pull_with_archive_flare.flare == {}
112+
assert merged_pull_with_archive_flare._flare is None
113+
assert merged_pull_with_archive_flare._flare_storage_path is None
114+
115+
mock_logs.reset_mock()
116+
# check that once these pulls are corrected they are not corrected again
117+
task = FlareCleanupTask()
118+
task.run_cron_task(transactional_db)
119+
120+
mock_logs.assert_has_calls(
121+
[
122+
call("Starting FlareCleanupTask"),
123+
call("FlareCleanupTask cleared 0 _flares"),
124+
call("FlareCleanupTask will clear 0 Archive flares"),
125+
call("FlareCleanupTask cleared 0 Archive flares"),
126+
]
127+
)

0 commit comments

Comments
 (0)