Add configuration for thresholds and runtime

Add PR changes

Author: susilnem

api/management/commands/cron_job_monitor.py

@@ -5,7 +5,7 @@
 from django.conf import settings
 from django.core.management.base import BaseCommand
 
-from main.sentry import SentryMonitor
+from main.sentry import SentryMonitor, SentryMonitorConfig
 from main.settings import SENTRY_DSN
 
 logger = logging.getLogger(__name__)
@@ -48,6 +48,10 @@ def handle(self, *args, **options):
                         "type": "crontab",
                         "value": str(schedule),
                     },
+                    "tz": settings.TIME_ZONE,
+                    "checkin_margin": SentryMonitorConfig.get_checkin_margin(cronjob),
+                    "failure_issue_threshold": SentryMonitorConfig.get_failure_issue_threshold(cronjob),
+                    "recovery_threshold": SentryMonitorConfig.get_recovery_threshold(cronjob),
                 },
                 "environment": settings.GO_ENVIRONMENT,
                 "status": "ok",

api/management/commands/index_and_notify.py

@@ -67,7 +67,6 @@
 }
 
 
-@monitor(monitor_slug=SentryMonitor.INDEX_AND_NOTIFY)
 class Command(BaseCommand):
     help = "Index and send notifications about new/changed records"
 
@@ -920,6 +919,7 @@ def check_ingest_issues(self, having_ingest_issue):
                 + ", notification sent to IM team"
             )
 
+    @monitor(monitor_slug=SentryMonitor.INDEX_AND_NOTIFY)
     def handle(self, *args, **options):
         if self.is_digest_mode():
             time_diff = self.diff_1_week()  # in digest mode (once a week, for new_entities only) we use a bigger interval

api/management/commands/ingest_appeals.py

@@ -33,7 +33,6 @@
 GEC_CODES = GECCode.objects.select_related("country").all()
 
 
-@monitor(monitor_slug=SentryMonitor.INGEST_APPEALS)
 class Command(BaseCommand):
     help = "Add new entries from Access database file"
 
@@ -292,6 +291,7 @@ def parse_appeal_record(self, r, **options):
 
         return fields
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_APPEALS)
     def handle(self, *args, **options):
         logger.info("Starting appeals ingest")
         start_appeals_count = Appeal.objects.all().count()

api/management/commands/ingest_disaster_law.py

@@ -8,10 +8,10 @@
 from main.sentry import SentryMonitor
 
 
-@monitor(monitor_slug=SentryMonitor.INGEST_DISASTER_LAW)
 class Command(BaseCommand):
     help = "Add ICRC data"
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_DISASTER_LAW)
     def handle(self, *args, **kwargs):
         logger.info("Starting Disaster Law data")
         home_url = "https://disasterlaw.ifrc.org/"

api/management/commands/ingest_icrc.py

@@ -8,10 +8,10 @@
 from main.sentry import SentryMonitor
 
 
-@monitor(monitor_slug=SentryMonitor.INGEST_ICRC)
 class Command(BaseCommand):
     help = "Add ICRC data"
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_ICRC)
     def handle(self, *args, **kwargs):
         logger.info("Strating ICRC data ingest")
         HEADERS = {

api/management/commands/ingest_ns_capacity.py

@@ -8,10 +8,10 @@
 from main.sentry import SentryMonitor
 
 
-@monitor(monitor_slug=SentryMonitor.INGEST_NS_CAPACITY)
 class Command(BaseCommand):
     help = "Add ns contact details"
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_NS_CAPACITY)
     def handle(self, *args, **kwargs):
         logger.info("Starting NS Contacts")
 

api/management/commands/ingest_ns_contact.py

@@ -5,6 +5,7 @@
 import xmltodict
 from django.conf import settings
 from django.core.management.base import BaseCommand
+from django.db import transaction
 from requests.auth import HTTPBasicAuth
 from sentry_sdk.crons import monitor
 
@@ -13,10 +14,11 @@
 from main.sentry import SentryMonitor
 
 
-@monitor(monitor_slug=SentryMonitor.INGEST_NS_CONTACT)
 class Command(BaseCommand):
     help = "Add ns contact details"
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_NS_CONTACT)
+    @transaction.atomic
     def handle(self, *args, **kwargs):
         logger.info("Starting NS Contacts")
         url = "https://go-api.ifrc.org/"

api/management/commands/ingest_ns_directory.py

@@ -2,6 +2,7 @@
 import xmltodict
 from django.conf import settings
 from django.core.management.base import BaseCommand
+from django.db import transaction
 from requests.auth import HTTPBasicAuth
 from sentry_sdk.crons import monitor
 
@@ -10,10 +11,11 @@
 from main.sentry import SentryMonitor
 
 
-@monitor(monitor_slug=SentryMonitor.INGEST_NS_DIRECTORY)
 class Command(BaseCommand):
     help = "Add ns contact details"
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_NS_DIRECTORY)
+    @transaction.atomic
     def handle(self, *args, **kwargs):
         def postprocessor(path, key, value):
             if key == "@i:nil":
@@ -41,6 +43,7 @@ def postprocessor(path, key, value):
             raise Exception("Error querying NationalSocietiesContacts")
 
         added = 0
+        created_country_directory_ids = []
         dict_data = xmltodict.parse(response.content, postprocessor=postprocessor)
         for data in dict_data["ArrayOfNationalSocietiesContacts"]["NationalSocietiesContacts"]:
             country_name = data["CON_country"] if isinstance(data["CON_country"], str) else None
@@ -62,14 +65,15 @@ def postprocessor(path, key, value):
                         "position": data["CON_title"],
                         "country": country,
                     }
-                    existing_qs = CountryDirectory.objects.filter(
+                    country_directory, _ = CountryDirectory.objects.get_or_create(
                         country=country,
                         first_name__iexact=data["first_name"],
                         last_name__iexact=data["last_name"],
                         position__iexact=data["position"],
                     )
-                    if not existing_qs.exists():
-                        CountryDirectory.objects.create(**data)
+                    created_country_directory_ids.append(country_directory.pk)
+        # NOTE: Deleting the country directory which are not available in the source
+        CountryDirectory.objects.exclude(id__in=created_country_directory_ids).delete()
         text_to_log = "%s Ns Directory added" % added
         logger.info(text_to_log)
         body = {"name": "ingest_ns_directory", "message": text_to_log, "num_result": added, "status": CronJobStatus.SUCCESSFUL}

api/management/commands/ingest_ns_document.py

@@ -5,17 +5,19 @@
 import requests
 from django.conf import settings
 from django.core.management.base import BaseCommand
+from django.db import transaction
 from sentry_sdk.crons import monitor
 
 from api.logger import logger
 from api.models import Country, CountryKeyDocument, CronJob, CronJobStatus
 from main.sentry import SentryMonitor
 
 
-@monitor(monitor_slug=SentryMonitor.INGEST_NS_DOCUMENT)
 class Command(BaseCommand):
     help = "Add ns documents"
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_NS_DOCUMENT)
+    @transaction.atomic
     def handle(self, *args, **kwargs):
         logger.info("Starting NS Key Documents")
 
@@ -98,6 +100,7 @@ def fetch_all_country_documents(self, api_key, country_table):
 
     def save_documents_to_database(self, result):
         added = 0
+        created_country_key_document_ids = []
         for document in result:
             country = Country.objects.filter(fdrs=document["country_code"]).first()
             if country is None:
@@ -122,8 +125,20 @@ def save_documents_to_database(self, result):
                 country_key_document.year = document["year"]
                 country_key_document.end_year = document["end_year"]
                 country_key_document.year_text = document["year_text"]
-                country_key_document.save(update_fields=["name", "thumbnail", "document_type", "year", "end_year", "year_text"])
+                country_key_document.save(
+                    update_fields=[
+                        "name",
+                        "thumbnail",
+                        "document_type",
+                        "year",
+                        "end_year",
+                        "year_text",
+                    ]
+                )
+            created_country_key_document_ids.append(country_key_document.pk)
             added += 1
+        # NOTE: Deleting the CountryKeyDocument that are not in the source
+        CountryKeyDocument.objects.exclude(id__in=created_country_key_document_ids).delete()
         return added
 
     def sync_cron_success(self, text_to_log, added):

api/management/commands/ingest_ns_initiatives.py

@@ -3,17 +3,19 @@
 import requests
 from django.conf import settings
 from django.core.management.base import BaseCommand
+from django.db import transaction
 from sentry_sdk.crons import monitor
 
 from api.logger import logger
 from api.models import Country, CronJob, CronJobStatus, NSDInitiatives
 from main.sentry import SentryMonitor
 
 
-@monitor(monitor_slug=SentryMonitor.INGEST_NS_INITIATIVES)
 class Command(BaseCommand):
     help = "Add ns initiatives"
 
+    @monitor(monitor_slug=SentryMonitor.INGEST_NS_INITIATIVES)
+    @transaction.atomic
     def handle(self, *args, **kwargs):
         logger.info("Starting NS Inititatives")
         api_key = settings.NS_INITIATIVES_API_KEY
@@ -43,6 +45,7 @@ def handle(self, *args, **kwargs):
             ],
         )
         funding_data = funding_data.replace({np.nan: None})
+        created_ns_initiatives_pk = []
         for data in funding_data.values.tolist():
             # TODO: Filter not by society name
             country = Country.objects.filter(society_name__iexact=data[0]).first()
@@ -60,11 +63,14 @@ def handle(self, *args, **kwargs):
                 )
                 if not created:
                     nsd_initiatives.title = data[3]
+                    nsd_initiatives.categories = data[4]
                     nsd_initiatives.allocation = data[5]
                     nsd_initiatives.funding_period = data[6]
-                    nsd_initiatives.categories = data[4]
-                    nsd_initiatives.save(update_fields=["title", "allocation", "funding_period", "categories"])
+                    nsd_initiatives.save(update_fields=["title", "categories", "allocation", "funding_period"])
+                created_ns_initiatives_pk.append(nsd_initiatives.pk)
                 added += 1
+        # NOTE: Delete the NSDInitiatives that are not in the source
+        NSDInitiatives.objects.exclude(id__in=created_ns_initiatives_pk).delete()
 
         text_to_log = "%s Ns initiatives added" % added
         logger.info(text_to_log)