Skip to content

Commit 9565094

Browse files
ruuushhhcursoragent
ruuushhh
and
cursoragent
authored
feat: add max attempts limit for stuck export re-trigger (#609)
* feat: add max attempts limit for stuck export re-trigger Added stuck_export_re_attempt_count field to TaskLog model to limit re-export attempts to max 2, preventing duplicate exports in accounting software. Co-authored-by: Cursor <cursoragent@cursor.com> * fix tests * test: add tests for re_export_stuck_exports function Added comprehensive tests covering: - No stuck exports scenario - Stuck export found and re-exported - Max attempts limit (stuck_export_re_attempt_count >= 2) - Test workspace exclusion - IN_PROGRESS status handling - Recently updated tasks not considered stuck - Tasks older than 7 days excluded - Attempt counter increment verification Co-authored-by: Cursor <cursoragent@cursor.com> * fix: remove report_title field from Expense fixture Co-authored-by: Cursor <cursoragent@cursor.com> --------- Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent 75afb1b commit 9565094

File tree

7 files changed

+351
-26
lines changed

7 files changed

+351
-26
lines changed

‎apps/internal/tasks.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
import logging
22
from datetime import datetime, timedelta, timezone
33

4-
from django.db.models import Q
5-
from apps.tasks.enums import TaskLogStatusEnum
4+
from django.db.models import F, Q
65
from django_q.models import OrmQ, Schedule
76
from fyle_accounting_library.fyle_platform.enums import ExpenseImportSourceEnum
87

98
from apps.fyle.actions import post_accounting_export_summary, update_failed_expenses
109
from apps.fyle.models import ExpenseGroup
10+
from apps.tasks.enums import TaskLogStatusEnum
1111
from apps.tasks.models import TaskLog
1212
from apps.workspaces.actions import export_to_xero
1313
from apps.workspaces.models import Workspace
@@ -27,7 +27,8 @@ def re_export_stuck_exports():
2727
updated_at__lt=datetime.now() - timedelta(minutes=60),
2828
updated_at__gt=datetime.now() - timedelta(days=7),
2929
expense_group_id__isnull=False,
30-
workspace_id__in=prod_workspace_ids
30+
workspace_id__in=prod_workspace_ids,
31+
stuck_export_re_attempt_count__lt=2
3132
)
3233
if task_logs.count() > 0:
3334
logger.info('Re-exporting stuck task_logs')
@@ -49,7 +50,7 @@ def re_export_stuck_exports():
4950
for expense_group in expense_groups:
5051
expenses.extend(expense_group.expenses.all())
5152
workspace_ids_list = list(workspace_ids)
52-
task_logs.update(status=TaskLogStatusEnum.FAILED, updated_at=datetime.now(timezone.utc), re_attempt_export=True)
53+
task_logs.update(status=TaskLogStatusEnum.FAILED, updated_at=datetime.now(timezone.utc), re_attempt_export=True, stuck_export_re_attempt_count=F('stuck_export_re_attempt_count') + 1)
5354
for workspace_id in workspace_ids_list:
5455
errored_expenses = [expense for expense in expenses if expense.workspace_id == workspace_id]
5556
update_failed_expenses(errored_expenses, True)

‎apps/tasks/migrations/0016_tasklog_stuck_export_re_attempt_count.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# Generated by Django 4.2.26 on 2026-02-02 13:02
2+
3+
from django.db import migrations, models
4+
5+
6+
class Migration(migrations.Migration):
7+
8+
dependencies = [
9+
('tasks', '0015_tasklog_is_attachment_upload_failed'),
10+
]
11+
12+
operations = [
13+
migrations.AddField(
14+
model_name='tasklog',
15+
name='stuck_export_re_attempt_count',
16+
field=models.IntegerField(default=0, help_text='Stuck export re-attempt count'),
17+
),
18+
]

‎apps/tasks/models.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ class TaskLog(models.Model):
6969
updated_at = models.DateTimeField(auto_now=True, help_text="Updated at datetime")
7070
triggered_by = models.CharField(max_length=255, help_text="Triggered by", null=True, choices=IMPORTED_FROM_CHOICES)
7171
re_attempt_export = models.BooleanField(default=False, help_text='Is re-attempt export')
72+
stuck_export_re_attempt_count = models.IntegerField(default=0, help_text='Stuck export re-attempt count')
7273
is_attachment_upload_failed = models.BooleanField(default=False, help_text='Is attachment upload failed')
7374

7475
class Meta:

‎tests/sql_fixtures/reset_db_fixtures/reset_db.sql‎

Lines changed: 22 additions & 22 deletions
Large diffs are not rendered by default.

‎tests/test_internal/__init__.py‎

Whitespace-only changes.

‎tests/test_internal/conftest.py‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
from datetime import datetime, timezone
2+
3+
import pytest
4+
5+
from apps.fyle.models import Expense, ExpenseGroup
6+
from apps.workspaces.models import Workspace
7+
8+
9+
@pytest.fixture
10+
def create_workspace_for_stuck_export(db):
11+
workspace, _ = Workspace.objects.update_or_create(
12+
id=100,
13+
defaults={
14+
'name': 'Production Workspace',
15+
'fyle_org_id': 'or_stuck_test',
16+
}
17+
)
18+
return workspace
19+
20+
21+
@pytest.fixture
22+
def create_test_workspace(db):
23+
workspace, _ = Workspace.objects.update_or_create(
24+
id=101,
25+
defaults={
26+
'name': 'Fyle For Demo Test',
27+
'fyle_org_id': 'or_test_workspace',
28+
}
29+
)
30+
return workspace
31+
32+
33+
@pytest.fixture
34+
def create_expense_group_with_expenses(db, create_workspace_for_stuck_export):
35+
workspace = create_workspace_for_stuck_export
36+
37+
expense = Expense.objects.create(
38+
workspace_id=workspace.id,
39+
expense_id='tx_stuck_test_1',
40+
employee_email='test@fyle.in',
41+
employee_name='Test Employee',
42+
category='Meals',
43+
sub_category='Team Meals',
44+
project='Test Project',
45+
expense_number='E/2024/01/T/1',
46+
claim_number='C/2024/01/R/1',
47+
amount=100.0,
48+
currency='USD',
49+
foreign_amount=100.0,
50+
foreign_currency='USD',
51+
settlement_id='setl_stuck_1',
52+
reimbursable=True,
53+
billable=False,
54+
state='APPROVED',
55+
vendor='Test Vendor',
56+
cost_center='Test Cost Center',
57+
purpose='Test expense',
58+
report_id='rp_stuck_report_1',
59+
spent_at=datetime.now(tz=timezone.utc),
60+
approved_at=datetime.now(tz=timezone.utc),
61+
expense_created_at=datetime.now(tz=timezone.utc),
62+
expense_updated_at=datetime.now(tz=timezone.utc),
63+
fund_source='PERSONAL',
64+
verified_at=datetime.now(tz=timezone.utc),
65+
custom_properties={},
66+
org_id=workspace.fyle_org_id,
67+
file_ids=[],
68+
accounting_export_summary={}
69+
)
70+
71+
expense_group = ExpenseGroup.objects.create(
72+
workspace_id=workspace.id,
73+
fund_source='PERSONAL',
74+
exported_at=None,
75+
description={
76+
'report_id': 'rp_stuck_report_1',
77+
'employee_email': 'test@fyle.in',
78+
}
79+
)
80+
expense_group.expenses.add(expense)
81+
82+
return expense_group

‎tests/test_internal/test_tasks.py‎

Lines changed: 223 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,223 @@
1+
from datetime import datetime, timedelta, timezone
2+
from unittest import mock
3+
4+
from apps.fyle.models import ExpenseGroup
5+
from apps.internal.tasks import re_export_stuck_exports
6+
from apps.tasks.models import TaskLog
7+
8+
9+
def test_no_stuck_exports(db, mocker):
10+
mock_export = mocker.patch('apps.internal.tasks.export_to_xero')
11+
mock_update_failed = mocker.patch('apps.internal.tasks.update_failed_expenses')
12+
mock_post_summary = mocker.patch('apps.internal.tasks.post_accounting_export_summary')
13+
14+
re_export_stuck_exports()
15+
16+
mock_export.assert_not_called()
17+
mock_update_failed.assert_not_called()
18+
mock_post_summary.assert_not_called()
19+
20+
21+
def test_stuck_export_found_and_reexported(
22+
db, mocker, create_workspace_for_stuck_export, create_expense_group_with_expenses
23+
):
24+
workspace = create_workspace_for_stuck_export
25+
expense_group = create_expense_group_with_expenses
26+
27+
stuck_time = datetime.now(tz=timezone.utc) - timedelta(minutes=90)
28+
task_log = TaskLog.objects.create(
29+
workspace_id=workspace.id,
30+
expense_group_id=expense_group.id,
31+
type='CREATING_BILL',
32+
status='ENQUEUED',
33+
stuck_export_re_attempt_count=0
34+
)
35+
TaskLog.objects.filter(id=task_log.id).update(updated_at=stuck_time)
36+
37+
mock_export = mocker.patch('apps.internal.tasks.export_to_xero')
38+
mock_update_failed = mocker.patch('apps.internal.tasks.update_failed_expenses')
39+
mock_post_summary = mocker.patch('apps.internal.tasks.post_accounting_export_summary')
40+
mocker.patch('apps.internal.tasks.OrmQ.objects.all', return_value=[])
41+
mocker.patch('apps.internal.tasks.Schedule.objects.filter', return_value=mock.Mock(filter=mock.Mock(return_value=mock.Mock(first=mock.Mock(return_value=None)))))
42+
43+
re_export_stuck_exports()
44+
45+
task_log.refresh_from_db()
46+
assert task_log.status == 'FAILED'
47+
assert task_log.re_attempt_export == True
48+
assert task_log.stuck_export_re_attempt_count == 1
49+
50+
mock_update_failed.assert_called_once()
51+
mock_post_summary.assert_called_once()
52+
mock_export.assert_called_once()
53+
54+
55+
def test_max_attempts_limit_excludes_task(
56+
db, mocker, create_workspace_for_stuck_export, create_expense_group_with_expenses
57+
):
58+
workspace = create_workspace_for_stuck_export
59+
expense_group = create_expense_group_with_expenses
60+
61+
stuck_time = datetime.now(tz=timezone.utc) - timedelta(minutes=90)
62+
task_log = TaskLog.objects.create(
63+
workspace_id=workspace.id,
64+
expense_group_id=expense_group.id,
65+
type='CREATING_BILL',
66+
status='ENQUEUED',
67+
stuck_export_re_attempt_count=2
68+
)
69+
TaskLog.objects.filter(id=task_log.id).update(updated_at=stuck_time)
70+
71+
mock_export = mocker.patch('apps.internal.tasks.export_to_xero')
72+
mock_update_failed = mocker.patch('apps.internal.tasks.update_failed_expenses')
73+
mock_post_summary = mocker.patch('apps.internal.tasks.post_accounting_export_summary')
74+
75+
re_export_stuck_exports()
76+
77+
mock_export.assert_not_called()
78+
mock_update_failed.assert_not_called()
79+
mock_post_summary.assert_not_called()
80+
81+
task_log.refresh_from_db()
82+
assert task_log.status == 'ENQUEUED'
83+
assert task_log.stuck_export_re_attempt_count == 2
84+
85+
86+
def test_test_workspace_excluded(db, mocker, create_test_workspace):
87+
workspace = create_test_workspace
88+
89+
expense_group = ExpenseGroup.objects.create(
90+
workspace_id=workspace.id,
91+
fund_source='PERSONAL',
92+
exported_at=None,
93+
)
94+
95+
stuck_time = datetime.now(tz=timezone.utc) - timedelta(minutes=90)
96+
task_log = TaskLog.objects.create(
97+
workspace_id=workspace.id,
98+
expense_group_id=expense_group.id,
99+
type='CREATING_BILL',
100+
status='ENQUEUED',
101+
stuck_export_re_attempt_count=0
102+
)
103+
TaskLog.objects.filter(id=task_log.id).update(updated_at=stuck_time)
104+
105+
mock_export = mocker.patch('apps.internal.tasks.export_to_xero')
106+
mock_update_failed = mocker.patch('apps.internal.tasks.update_failed_expenses')
107+
108+
re_export_stuck_exports()
109+
110+
mock_export.assert_not_called()
111+
mock_update_failed.assert_not_called()
112+
113+
task_log.refresh_from_db()
114+
assert task_log.status == 'ENQUEUED'
115+
116+
117+
def test_in_progress_status_also_considered_stuck(
118+
db, mocker, create_workspace_for_stuck_export, create_expense_group_with_expenses
119+
):
120+
workspace = create_workspace_for_stuck_export
121+
expense_group = create_expense_group_with_expenses
122+
123+
stuck_time = datetime.now(tz=timezone.utc) - timedelta(minutes=90)
124+
task_log = TaskLog.objects.create(
125+
workspace_id=workspace.id,
126+
expense_group_id=expense_group.id,
127+
type='CREATING_BILL',
128+
status='IN_PROGRESS',
129+
stuck_export_re_attempt_count=0
130+
)
131+
TaskLog.objects.filter(id=task_log.id).update(updated_at=stuck_time)
132+
133+
mocker.patch('apps.internal.tasks.export_to_xero')
134+
mocker.patch('apps.internal.tasks.update_failed_expenses')
135+
mocker.patch('apps.internal.tasks.post_accounting_export_summary')
136+
mocker.patch('apps.internal.tasks.OrmQ.objects.all', return_value=[])
137+
mocker.patch('apps.internal.tasks.Schedule.objects.filter', return_value=mock.Mock(filter=mock.Mock(return_value=mock.Mock(first=mock.Mock(return_value=None)))))
138+
139+
re_export_stuck_exports()
140+
141+
task_log.refresh_from_db()
142+
assert task_log.status == 'FAILED'
143+
assert task_log.stuck_export_re_attempt_count == 1
144+
145+
146+
def test_task_updated_recently_not_considered_stuck(
147+
db, mocker, create_workspace_for_stuck_export, create_expense_group_with_expenses
148+
):
149+
workspace = create_workspace_for_stuck_export
150+
expense_group = create_expense_group_with_expenses
151+
152+
recent_time = datetime.now(tz=timezone.utc) - timedelta(minutes=30)
153+
task_log = TaskLog.objects.create(
154+
workspace_id=workspace.id,
155+
expense_group_id=expense_group.id,
156+
type='CREATING_BILL',
157+
status='ENQUEUED',
158+
stuck_export_re_attempt_count=0
159+
)
160+
TaskLog.objects.filter(id=task_log.id).update(updated_at=recent_time)
161+
162+
mock_export = mocker.patch('apps.internal.tasks.export_to_xero')
163+
164+
re_export_stuck_exports()
165+
166+
mock_export.assert_not_called()
167+
168+
task_log.refresh_from_db()
169+
assert task_log.status == 'ENQUEUED'
170+
171+
172+
def test_task_older_than_7_days_not_considered(
173+
db, mocker, create_workspace_for_stuck_export, create_expense_group_with_expenses
174+
):
175+
workspace = create_workspace_for_stuck_export
176+
expense_group = create_expense_group_with_expenses
177+
178+
old_time = datetime.now(tz=timezone.utc) - timedelta(days=10)
179+
task_log = TaskLog.objects.create(
180+
workspace_id=workspace.id,
181+
expense_group_id=expense_group.id,
182+
type='CREATING_BILL',
183+
status='ENQUEUED',
184+
stuck_export_re_attempt_count=0
185+
)
186+
TaskLog.objects.filter(id=task_log.id).update(updated_at=old_time)
187+
188+
mock_export = mocker.patch('apps.internal.tasks.export_to_xero')
189+
190+
re_export_stuck_exports()
191+
192+
mock_export.assert_not_called()
193+
194+
task_log.refresh_from_db()
195+
assert task_log.status == 'ENQUEUED'
196+
197+
198+
def test_attempt_count_increments_on_each_retry(
199+
db, mocker, create_workspace_for_stuck_export, create_expense_group_with_expenses
200+
):
201+
workspace = create_workspace_for_stuck_export
202+
expense_group = create_expense_group_with_expenses
203+
204+
stuck_time = datetime.now(tz=timezone.utc) - timedelta(minutes=90)
205+
task_log = TaskLog.objects.create(
206+
workspace_id=workspace.id,
207+
expense_group_id=expense_group.id,
208+
type='CREATING_BILL',
209+
status='ENQUEUED',
210+
stuck_export_re_attempt_count=1
211+
)
212+
TaskLog.objects.filter(id=task_log.id).update(updated_at=stuck_time)
213+
214+
mocker.patch('apps.internal.tasks.export_to_xero')
215+
mocker.patch('apps.internal.tasks.update_failed_expenses')
216+
mocker.patch('apps.internal.tasks.post_accounting_export_summary')
217+
mocker.patch('apps.internal.tasks.OrmQ.objects.all', return_value=[])
218+
mocker.patch('apps.internal.tasks.Schedule.objects.filter', return_value=mock.Mock(filter=mock.Mock(return_value=mock.Mock(first=mock.Mock(return_value=None)))))
219+
220+
re_export_stuck_exports()
221+
222+
task_log.refresh_from_db()
223+
assert task_log.stuck_export_re_attempt_count == 2

0 commit comments

Comments
 (0)