diff --git a/apps/common/tasks.py b/apps/common/tasks.py index d56229f3..871b70ff 100644 --- a/apps/common/tasks.py +++ b/apps/common/tasks.py @@ -18,3 +18,8 @@ def clear_expired_django_sessions(): logger.warning("Clear expired django sessions") return management.call_command("clearsessions", verbosity=0) + + +@shared_task +def celery_queue_uptime_check(queue: str): + logger.info("Celery Queue %s is taking task...", queue) diff --git a/apps/contributor/graphql/types.py b/apps/contributor/graphql/types.py index a4b6af19..4dd8f64f 100644 --- a/apps/contributor/graphql/types.py +++ b/apps/contributor/graphql/types.py @@ -82,6 +82,7 @@ class ContributorUserGroupType(UserResourceTypeMixin, ArchivableResourceTypeMixi models.Subquery( ContributorUserGroupMembership.objects.filter( user_group_id=models.OuterRef("id"), + is_active=True, ) .order_by() .values("user_group_id") diff --git a/apps/existing_database/management/commands/loaddata_from_existing_database.py b/apps/existing_database/management/commands/loaddata_from_existing_database.py index b4358823..e9903f16 100644 --- a/apps/existing_database/management/commands/loaddata_from_existing_database.py +++ b/apps/existing_database/management/commands/loaddata_from_existing_database.py @@ -506,7 +506,7 @@ def create_project( existing_project: existing_db_models.Project, requesting_organization: str, bot_user: User, -): +) -> tuple[Project, bool]: try: assert existing_project.project_type is not None, "Project type should be defined" @@ -544,7 +544,7 @@ def create_project( # 2022-10-13 00:00:00 -> 2022-10-13 project_metadata["last_contribution_date"] = day_.split(" ")[0] - return Project.objects.update_or_create( + project, project_created = Project.objects.update_or_create( old_id=existing_project.project_id, create_defaults=dict( client_id=client_id, @@ -553,6 +553,15 @@ def create_project( ), defaults=project_metadata, ) + + # NOTE: Django doesn't allow custom value for auto_add fields, we can use objects.update to do that + if project_creation_date := parse_datetime(existing_project.created): + Project.objects.filter(id=project.pk).update( + created_at=project_creation_date, + modified_at=project_creation_date, + ) + + return project, project_created except IntegrityError as e: if not str(e).startswith('duplicate key value violates unique constraint "unique_project_name"'): raise diff --git a/apps/mapping/firebase/pull.py b/apps/mapping/firebase/pull.py index 39ee438d..62c7c484 100644 --- a/apps/mapping/firebase/pull.py +++ b/apps/mapping/firebase/pull.py @@ -4,6 +4,7 @@ from main.bulk_managers import BulkCreateManager from main.config import Config from main.logging import log_extra +from main.sentry import SentryTag from .utils import ( FirebaseCleanup, @@ -22,12 +23,11 @@ def _transfer_results_for_project( firebase_cleanup: FirebaseCleanup, project: Project, ): + SentryTag.set_tags({SentryTag.Tag.PROJECT: project.pk}) group_results = FH.ref(Config.FirebaseKeys.results_project_groups(project.firebase_id)).get() assert type(group_results) is dict results_to_temp_table(bulk_create_manager, firebase_cleanup, project, group_results) - # generate user_group mapping session - def pull_results_from_firebase(): # TODO: Add a lock to prevent concurrent execution. We have lock within celery @@ -59,6 +59,7 @@ def pull_results_from_firebase(): { "tutorial_firebase_id": project_firebase_id, }, + sentry_tags={SentryTag.Tag.PROJECT: project_firebase_id}, ), ) firebase_cleanup.add_project(project_firebase_id=project_firebase_id) @@ -73,9 +74,11 @@ def pull_results_from_firebase(): { "project_firebase_id": project_firebase_id, }, + sentry_tags={SentryTag.Tag.PROJECT: project_firebase_id}, ), ) - firebase_cleanup.add_project(project_firebase_id=project_firebase_id) + # TODO: Cleanup this.. For now let's do this manually + # firebase_cleanup.add_project(project_firebase_id=project_firebase_id) continue _transfer_results_for_project( diff --git a/apps/project/exports/overall_stats.py b/apps/project/exports/overall_stats.py index cc8ce210..c620380f 100644 --- a/apps/project/exports/overall_stats.py +++ b/apps/project/exports/overall_stats.py @@ -92,7 +92,7 @@ def regenerate_projects_csv(temp_projects_csv: typing.IO): # type: ignore[repor "area_sqkm": models.F("aoi_geometry__total_area"), "centroid": None, # TODO: use this after removing from model models.F("aoi_geometry__centroid"), "geom": models.F("aoi_geometry__geometry"), - "progress": None, + "progress": None, # NOTE: This is changed to float later "number_of_contributor_users": None, "number_of_results": None, "number_of_results_for_progress": None, @@ -116,6 +116,7 @@ def regenerate_projects_csv(temp_projects_csv: typing.IO): # type: ignore[repor data["image_url"] = image_file_url data["status_display"] = ProjectStatusEnum(data["status"]).label data["project_type_display"] = ProjectTypeEnum(data["project_type"]).label + data["progress"] = data["progress"] / 100 writer.writerow(data) diff --git a/apps/project/graphql/types/types.py b/apps/project/graphql/types/types.py index 7c4f72d1..a20a0790 100644 --- a/apps/project/graphql/types/types.py +++ b/apps/project/graphql/types/types.py @@ -176,12 +176,17 @@ class ProjectType(UserResourceTypeMixin, ProjectExportAssetTypeMixin, FirebasePu aoi_geometry: GeometryType | None progress_status: strawberry.auto - progress: strawberry.auto number_of_contributor_users: strawberry.auto number_of_results: strawberry.auto number_of_results_for_progress: strawberry.auto last_contribution_date: strawberry.auto + @strawberry_django.field( + description=str(Project._meta.get_field("progress").help_text), # type: ignore[reportAttributeAccessIssue] + ) + def progress(self, project: strawberry.Parent[Project]) -> float: + return project.progress / 100 + @strawberry_django.field( description="No. of unique contributors in this project", ) diff --git a/assets b/assets index 3a2cafe0..555b47f6 160000 --- a/assets +++ b/assets @@ -1 +1 @@ -Subproject commit 3a2cafe08383e86a0889b2305001f5f854c06ac9 +Subproject commit 555b47f659791a5b873f55bf9771be2bdf94ef9a diff --git a/main/celery.py b/main/celery.py index dc3e0027..1d872d16 100644 --- a/main/celery.py +++ b/main/celery.py @@ -8,9 +8,8 @@ from celery import signals from django.conf import settings from django.db import models -from kombu import Queue -from .cronjobs import BEAT_SCHEDULES +from .cronjobs import BEAT_SCHEDULES, CeleryQueue if TYPE_CHECKING: from celery.app.task import Task @@ -24,13 +23,6 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "main.settings") -class CeleryQueue: - # NOTE: Make sure all queue names are lowercase (They are in k8s) - default = Queue("default") - - ALL_QUEUE = (default,) - - class Celery(celery.Celery): def on_configure(self): # type: ignore[reportIncompatibleVariableOverride] if settings.SENTRY_ENABLED: diff --git a/main/cronjobs.py b/main/cronjobs.py index 1ba4d740..2d3d9bc1 100644 --- a/main/cronjobs.py +++ b/main/cronjobs.py @@ -2,6 +2,7 @@ from datetime import datetime from celery.schedules import crontab +from kombu import Queue from sentry_sdk.integrations.celery import beat as sentry_celery_beat if typing.TYPE_CHECKING: @@ -20,6 +21,13 @@ class TimeConstants: EVERY_1_MINUTES = crontab(minute="*/1") +class CeleryQueue: + # NOTE: Make sure all queue names are lowercase (They are in k8s) + default = Queue("default") + + ALL_QUEUE = (default,) + + class CronJobOption(typing.TypedDict, total=False): # https://docs.celeryq.dev/en/latest/reference/celery.app.task.html#celery.app.task.Task.apply_async @@ -37,6 +45,7 @@ class CeleryBeatSchedule(typing.TypedDict): task: str schedule: crontab options: CronJobOption + args: tuple[typing.Any, ...] | None class CronJobSentryConfig(typing.NamedTuple): @@ -67,6 +76,7 @@ def as_dict(self) -> "MonitorConfig": class CronJob(typing.NamedTuple): task: str schedule: crontab + args: tuple[typing.Any, ...] | None = None sentry_config: CronJobSentryConfig = CronJobSentryConfig() options: CronJobOption = {} @@ -118,11 +128,27 @@ class CronJob(typing.NamedTuple): max_runtime=2, ), ), + # Queue uptime + **{ + f"celery_queue_uptime_{celery_queue.name}": CronJob( + task="apps.common.tasks.celery_queue_uptime_check", + args=(celery_queue.name,), + schedule=TimeConstants.EVERY_HOUR, + options=CronJobOption(expires=TimeConstants.SECONDS_IN_A_HOUR), + sentry_config=CronJobSentryConfig( + failure_issue_threshold=2, + checkin_margin=2, + max_runtime=2, + ), + ) + for celery_queue in CeleryQueue.ALL_QUEUE + }, } BEAT_SCHEDULES: dict[str, CeleryBeatSchedule] = { name: { "task": config.task, + "args": config.args, "schedule": config.schedule, "options": config.options, } diff --git a/main/logging.py b/main/logging.py index 7d75c3bd..bbddd49b 100644 --- a/main/logging.py +++ b/main/logging.py @@ -3,6 +3,8 @@ import requests +from main.sentry import SentryTag + def log_render_extra_context(record: logging.LogRecord): """Append extra->context to logs. @@ -20,10 +22,14 @@ def log_render_extra_context(record: logging.LogRecord): return True -def log_extra(extra: dict[typing.Any, typing.Any]): +def log_extra( + extra: dict[typing.Any, typing.Any], + sentry_tags: dict[SentryTag.Tag, typing.Any] | None = None, +): """Basic helper function to view extra argument in logs using log_render_extra_context.""" return { "context": extra, + "tags": sentry_tags, # https://forum.sentry.io/t/how-to-add-tags-to-python-logging/323/4 } diff --git a/main/sentry.py b/main/sentry.py index 18be4f5e..bd987f3a 100644 --- a/main/sentry.py +++ b/main/sentry.py @@ -1,6 +1,7 @@ import dataclasses import json import typing +from enum import Enum import sentry_sdk from asgiref.sync import sync_to_async @@ -106,3 +107,16 @@ def track_transaction(graphql_urls: set[str], request: typing.Any): "is_superuser": user.is_superuser, }, ) + + +class SentryTag: + """https://docs.sentry.io/platforms/python/enriching-events/tags/.""" + + class Tag(Enum): + _BASE = "mapswipe." + PROJECT = _BASE + "project" + + @staticmethod + def set_tags(kwargs: dict[Tag, int | str]): + for key, value in kwargs.items(): + sentry_sdk.set_tag(key.value, value) diff --git a/pyproject.toml b/pyproject.toml index 01f2351b..4456f461 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -242,6 +242,9 @@ reportPrivateImportUsage = "error" [tool.coverage.report] # https://coverage.readthedocs.io/en/7.10.6/excluding.html#advanced-exclusion exclude_also = [ - 'raise NotImplementedError', - 'if typing.TYPE_CHECKING:', + "raise NotImplementedError", + "if typing.TYPE_CHECKING:", +] +partial_branches = [ + "if typing.TYPE_CHECKING:", ] diff --git a/schema.graphql b/schema.graphql index 46edb314..411a0b94 100644 --- a/schema.graphql +++ b/schema.graphql @@ -1802,7 +1802,7 @@ type ProjectType implements UserResourceTypeMixin & ProjectExportAssetTypeMixin processingStatus: ProjectProcessingStatusEnum """Percentage of the required contribution that has been completed""" - progress: Int! + progress: Float! progressStatus: ProjectProgressStatusEnum! """Provide project instruction"""