Skip to content

Handle timeout error in background jobs #192

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tnagorra merged 4 commits into from
Oct 10, 2025
Merged

Handle timeout error in background jobs #192

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5b9c3f5
fix(error): handle timeout error in background jobs
tnagorra Oct 9, 2025
b05f837
fix(project)!: re-use contributors count logic
tnagorra Oct 9, 2025
78712e7
fix(error): convert pydantic type to dict recursively in errors
tnagorra Oct 9, 2025
2dff549
chore(submodules): update submodules to latest stable
tnagorra Oct 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 0 additions & 7 deletions apps/project/graphql/types/types.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,13 +187,6 @@ class ProjectType(UserResourceTypeMixin, ProjectExportAssetTypeMixin, FirebasePu
def progress(self, project: strawberry.Parent[Project]) -> float:
return project.progress / 100

@strawberry_django.field(
description="No. of unique contributors in this project",
)
def contributors_count(self) -> int:
# TODO(tnagorra): We need to implement this
return 0

@strawberry_django.field(
only=["topic", "region", "project_number", "requesting_organization__name", "project_type"],
annotate={"generated_name": Project.generate_name_query()},
Expand Down
101 changes: 74 additions & 27 deletions apps/tutorial/tests/mutation_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@

from django.core.files.temp import NamedTemporaryFile
from PIL import Image
from ulid import ULID

from apps.project.factories import OrganizationFactory, ProjectFactory
from apps.project.models import (
Expand Down Expand Up @@ -346,7 +345,7 @@ def _update_tutorial_status_mutation(self, pk: str, tutorial_data: dict, **kwarg

def test_tutorial_create(self):
tutorial_data = {
"clientId": str(ULID()),
"clientId": "01K748SHD9W5BTQA8EB9RCZGB7",
"name": "My Tutorial",
"project": self.project.pk,
}
Expand Down Expand Up @@ -378,21 +377,22 @@ def test_tutorial_create(self):
# Creating Project Image Asset
tutorial_asset_data = {
"tutorial": latest_tutorial.pk,
"clientId": str(ULID()),
"clientId": "01K748TDZPSDKPX1QVC5J5H558",
}
content = self._create_tutorial_image_asset(tutorial_asset_data)
resp_data = content["data"]["createTutorialAsset"]
assert resp_data["errors"] is None, content
image_asset = resp_data["result"]

# Update Tutorial
tutorial_data.pop("project")

tutorial_data = {
**tutorial_data,
"scenarios": [
{
"create": {
"clientId": str(ULID()),
"clientId": "01K748TDZPADVS0XX5FV59Q8JC",
"scenarioPageNumber": 1,
"instructionsDescription": "Anything that is not naturally occurring",
"instructionsIcon": "STAR_OUTLINE",
Expand All @@ -405,32 +405,32 @@ def test_tutorial_create(self):
"successTitle": "Well done!",
"tasks": [
{
"clientId": str(ULID()),
"clientId": "01K748TDZQAX1242QFT3R8V3H1",
"projectTypeSpecifics": {"find": {"tileZ": 18, "tileX": 193196, "tileY": 110087}},
"reference": 0,
},
{
"clientId": str(ULID()),
"clientId": "01K748TDZQDAHN2RAEJ6KKQZBS",
"projectTypeSpecifics": {"find": {"tileZ": 18, "tileX": 193196, "tileY": 110088}},
"reference": 1,
},
{
"clientId": str(ULID()),
"clientId": "01K748TDZQN9KR7PPAWY282B8V",
"projectTypeSpecifics": {"find": {"tileZ": 18, "tileX": 193196, "tileY": 110089}},
"reference": 0,
},
{
"clientId": str(ULID()),
"clientId": "01K748TDZQCYF1Q88MVW3B5J8A",
"projectTypeSpecifics": {"find": {"tileZ": 18, "tileX": 193197, "tileY": 110087}},
"reference": 0,
},
{
"clientId": str(ULID()),
"clientId": "01K748TDZR861B76GJAYS3WCY6",
"projectTypeSpecifics": {"find": {"tileZ": 18, "tileX": 193197, "tileY": 110088}},
"reference": 1,
},
{
"clientId": str(ULID()),
"clientId": "01K748TDZRT35XEZFT0SRR1EF0",
"projectTypeSpecifics": {"find": {"tileZ": 18, "tileX": 193197, "tileY": 110089}},
"reference": 0,
},
Expand All @@ -439,7 +439,7 @@ def test_tutorial_create(self):
},
{
"create": {
"clientId": str(ULID()),
"clientId": "01K748TDZSP64X7DY58NQZQ87E",
"scenarioPageNumber": 2,
"instructionsDescription": "Anything that is not naturally occurring",
"instructionsIcon": "STAR_OUTLINE",
Expand All @@ -452,32 +452,32 @@ def test_tutorial_create(self):
"successTitle": "Well done!",
"tasks": [
{
"clientId": str(ULID()),
"clientId": "01K748TDZSBB9R1D0B0JYWQVHS",
"projectTypeSpecifics": {"find": {"tileZ": 18, "tileX": 193204, "tileY": 110087}},
"reference": 1,
},
{
"clientId": str(ULID()),
"clientId": "01K748TDZS4875A6W77FFKCB39",
"projectTypeSpecifics": {"find": {"tileZ": 18, "tileX": 193204, "tileY": 110088}},
"reference": 1,
},
{
"clientId": str(ULID()),
"clientId": "01K748TDZSDZJ7VPY35KQRMZ8E",
"projectTypeSpecifics": {"find": {"tileZ": 18, "tileX": 193204, "tileY": 110089}},
"reference": 1,
},
{
"clientId": str(ULID()),
"clientId": "01K748TDZS86H58VNJ876JTB7R",
"projectTypeSpecifics": {"find": {"tileZ": 18, "tileX": 193205, "tileY": 110087}},
"reference": 1,
},
{
"clientId": str(ULID()),
"clientId": "01K748TDZS5DGYD7RGHT2EEEV6",
"projectTypeSpecifics": {"find": {"tileZ": 18, "tileX": 193205, "tileY": 110088}},
"reference": 1,
},
{
"clientId": str(ULID()),
"clientId": "01K748TDZSC0NVNAMZKBQRRYQY",
"projectTypeSpecifics": {"find": {"tileZ": 18, "tileX": 193205, "tileY": 110089}},
"reference": 1,
},
Expand All @@ -488,26 +488,26 @@ def test_tutorial_create(self):
"informationPages": [
{
"create": {
"clientId": str(ULID()),
"clientId": "01K748TDZTPW7Y19MEMPH0332S",
"title": "Man-made structures",
"pageNumber": 1,
"blocks": [
{
"clientId": str(ULID()),
"clientId": "01K748TDZTPVP7HF5VX03HTW9Z",
"blockNumber": 1,
"blockType": "TEXT",
"text": "Man-made structures are physical constructions created by humans, typically "
"using tools, materials, and engineering principles.",
},
{
"clientId": str(ULID()),
"clientId": "01K748TDZT5T9D33D78VV5NFKS",
"blockNumber": 2,
"blockType": "TEXT",
"text": "These structures are built to serve specific purposes, such as housing, "
"transportation, defense, communication, or recreation.",
},
{
"clientId": str(ULID()),
"clientId": "01K748TDZT7PV91ZX9W5H4BS47",
"blockNumber": 3,
"blockType": "IMAGE",
"image": image_asset["id"],
Expand All @@ -517,12 +517,12 @@ def test_tutorial_create(self):
},
{
"create": {
"clientId": str(ULID()),
"clientId": "01K748TDZTZ2M1FJWEKHGDVSV9",
"title": "Natural structures",
"pageNumber": 2,
"blocks": [
{
"clientId": str(ULID()),
"clientId": "01K748TDZTS0MK857GPZHKYDWW",
"blockNumber": 1,
"blockType": "TEXT",
"text": "Natural structures are physical formations that are created by nature "
Expand All @@ -533,7 +533,6 @@ def test_tutorial_create(self):
},
],
}
tutorial_data.pop("project")

# Updating Tutorial: Without Authentication
self.logout()
Expand All @@ -547,8 +546,56 @@ def test_tutorial_create(self):
},
], content

# Updating Tutorial: With Authentication
# Updating Tutorial: Check deep nested validation issues
self.force_login(self.user)
tutorial_data["scenarios"][1]["create"]["tasks"][1]["reference"] = -9 # type: ignore[reportArgumentType, reportIndexIssue]
content = self._update_tutorial_mutation(str(latest_tutorial.pk), tutorial_data)
tutorial_data["scenarios"][1]["create"]["tasks"][1]["reference"] = 1 # type: ignore[reportArgumentType, reportIndexIssue]

resp_data = content["data"]["updateTutorial"]
expected_error_message = [
{
"field": "scenarios",
"client_id": "01K748SHD9W5BTQA8EB9RCZGB7",
"messages": None,
"object_errors": None,
"array_errors": [
{
"client_id": "01K748TDZSP64X7DY58NQZQ87E",
"messages": None,
"object_errors": [
{
"field": "tasks",
"client_id": "01K748TDZSP64X7DY58NQZQ87E",
"messages": None,
"object_errors": None,
"array_errors": [
{
"client_id": "01K748TDZS4875A6W77FFKCB39",
"messages": None,
"object_errors": [
{
"field": "reference",
"client_id": "01K748TDZS4875A6W77FFKCB39",
"messages": "Ensure this value is greater than or equal to 0.",
"object_errors": None,
"array_errors": None,
"pydantic_errors": None,
},
],
},
],
"pydantic_errors": None,
},
],
},
],
"pydantic_errors": None,
},
]
assert resp_data["errors"] == expected_error_message, content

# Updating Tutorial: With Authentication and correct data
content = self._update_tutorial_mutation(str(latest_tutorial.pk), tutorial_data)
resp_data = content["data"]["updateTutorial"]
assert resp_data["errors"] is None, content
Expand Down Expand Up @@ -618,7 +665,7 @@ def get_update_for_task(tut: dict): # type: ignore[reportMissingTypeArgument]
},
{
"create": {
"clientId": str(ULID()),
"clientId": "01K748TDZTRDF6Z3X2VTNKJEGF",
# NOTE: blockNumber 2 is previously deleted so should not error on unique constraint
"blockNumber": 2,
"blockType": "TEXT",
Expand Down Expand Up @@ -744,7 +791,7 @@ def get_update_for_task(tut: dict): # type: ignore[reportMissingTypeArgument]
def test_tutorial_state_transitions(self):
# Create a draft tutorial
tutorial = TutorialFactory.create(
client_id=str(ULID()),
client_id="01K748TDZT84RRP5GACAYB71PP",
name="Status Tutorial",
project=self.project,
created_by=self.user,
Expand Down
2 changes: 1 addition & 1 deletion assets
Open in desktop
Submodule assets updated 0 files
2 changes: 1 addition & 1 deletion firebase
Open in desktop
Submodule firebase updated 1 files
+32 −1 .github/workflows/ci.yml
29 changes: 16 additions & 13 deletions project_types/base/project.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from abc import ABC, abstractmethod
from collections import defaultdict

from celery.exceptions import SoftTimeLimitExceeded
from django.contrib.gis.db.models import GeometryField
from django.contrib.gis.db.models.functions import Area
from django.core.files.base import ContentFile
Expand All @@ -18,7 +19,6 @@

from apps.common.models import FirebasePushStatusEnum
from apps.common.utils import get_absolute_uri
from apps.mapping.models import MappingSession
from apps.project.models import (
Project,
ProjectAsset,
Expand Down Expand Up @@ -226,6 +226,13 @@ def process_project(self):
exc_info=True,
)
self.project.status_message = str(ex)
elif isinstance(ex, SoftTimeLimitExceeded):
logger.error(
"process_project failed due to SoftTimeLimitExceeded",
extra=log_extra({"project": self.project.pk}),
exc_info=True,
)
self.project.status_message = "Project processing timed out before completion"
else:
logger.error(
"process_project failed",
Expand Down Expand Up @@ -421,17 +428,6 @@ def update_project_on_firebase(self, project_ref: FbReference, fb_project: fireb
assert self.project.tutorial_id is not None, "Tutorial is required before project can be pushed to firebase"
assert self.project.tutorial is not None, "Tutorial is required before project can be pushed to firebase"

unique_contributors_count = (
MappingSession.objects.filter(
project_task_group__in=ProjectTaskGroup.objects.filter(project=self.project),
)
.values(
"contributor_user_id",
)
.distinct()
.count()
)

project_ref.update(
value=firebase_utils.serialize(
firebase_models.FbProjectUpdateInput(
Expand All @@ -450,7 +446,7 @@ def update_project_on_firebase(self, project_ref: FbReference, fb_project: fireb
tutorialId=self.project.tutorial.firebase_id,
status=BaseProject.get_firebase_status(self.project.status_enum, not self.project.team_id),
teamId=self.project.team.firebase_id if self.project.team else None,
contributorCount=unique_contributors_count,
contributorCount=self.project.number_of_contributor_users,
progress=self.project.progress,
# FIXME(tnagorra): Need to check how we get this?
language="en-us",
Expand Down Expand Up @@ -514,6 +510,13 @@ def push_project_on_firebase(self):
exc_info=True,
)
self.project.status_message = str(ex)
elif isinstance(ex, SoftTimeLimitExceeded):
logger.error(
"push_to_firebase for project failed due to SoftTimeLimitExceeded",
extra=log_extra({"project": self.project.pk}),
exc_info=True,
)
self.project.status_message = "Project sync to firebase timed out before completion"
else:
logger.error(
"push_to_firebase for project failed",
Expand Down
31 changes: 25 additions & 6 deletions project_types/base/tutorial.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import typing
from abc import ABC, abstractmethod

from celery.exceptions import SoftTimeLimitExceeded
from django.core.files.base import ContentFile
from firebase_admin.db import Reference as FbReference # type: ignore[reportMissingTypeStubs]
from pydantic import BaseModel, ConfigDict
Expand Down Expand Up @@ -367,12 +368,30 @@ def push_tutorial_on_firebase(self):
try:
self._push_tutorial_on_firebase()
except Exception as ex:
logger.error(
"push_to_firebase for tutorial failed",
extra=log_extra({"tutorial": self.tutorial.pk}),
exc_info=True,
)
self.tutorial.status_message = str(ex) if isinstance(ex, TutorialValidationException) else None
if isinstance(ex, TutorialValidationException):
logger.warning(
"push_to_firebase for tutorial failed",
extra=log_extra({"tutorial": self.tutorial.pk}),
exc_info=True,
)
self.tutorial.status_message = str(ex)
elif isinstance(ex, SoftTimeLimitExceeded):
logger.error(
"push_to_firebase for tutorial failed due to SoftTimeLimitExceeded",
extra=log_extra({"tutorial": self.tutorial.pk}),
exc_info=True,
)
self.tutorial.status_message = "Tutorial sync to firebase timed out before completion"
else:
logger.error(
"push_to_firebase for tutorial failed",
extra=log_extra({"tutorial": self.tutorial.pk}),
exc_info=True,
)
self.tutorial.status_message = (
"Something unexpected happened! Please reach out on the MapSwipe slack for any further assistance."
)

self.tutorial.update_firebase_push_status(FirebasePushStatusEnum.FAILED, False)
# TODO(tnagorra): We also need to clear any intermediate values for groups, tasks and tutorial in firebase
# NOTE: If tutorial has already been published, we cannot update it's status
Expand Down
Loading
Loading