Merge pull request #2068 from IFRCGo/fix/country-page

Fix/country page

Author: samshara

Population.GO.Admin1.Matched.xlsx

@@ -63,7 +63,6 @@
     UserCountry,
     CountryOfFieldReportToReview,
     Export,
-    GDACSEvent,
     CountryKeyDocument,
     AppealType,
     CountrySupportingPartner
@@ -123,7 +122,6 @@
     GoHistoricalSerializer,
     CountryOfFieldReportToReviewSerializer,
     ExportSerializer,
-    GDACSEventSerializer,
     CountryKeyDocumentSerializer,
     CountryKeyFigureInputSerializer,
     CountryDisasterTypeCountSerializer,
@@ -149,7 +147,6 @@
     AppealDocumentFilter,
     FieldReportFilter,
     GoHistoricalFilter,
-    GDACSEventFileterSet,
     CountryKeyDocumentFilter,
     CountrySupportingPartnerFilter
 )
@@ -1272,16 +1269,6 @@ def get_queryset(self):
         return Export.objects.filter(requested_by=user).distinct()
 
 
-class GDACSEventViewSet(viewsets.ReadOnlyModelViewSet):
-    serializer_class = GDACSEventSerializer
-    filterset_class = GDACSEventFileterSet
-
-    def get_queryset(self):
-        today = timezone.now()
-        thirty_days_before = today + timedelta(days=-30)
-        return GDACSEvent.objects.filter(publication_date__gte=thirty_days_before)
-
-
 class CountrySupportingPartnerViewSet(viewsets.ModelViewSet):
     serializer_class = CountrySupportingPartnerSerializer
     filterset_class = CountrySupportingPartnerFilter

api/drf_views.py

@@ -19,7 +19,6 @@
     AppealHistory,
     AppealDocument,
     FieldReport,
-    GDACSEvent,
     CountryKeyDocument,
     CountrySupportingPartner
 
@@ -332,27 +331,6 @@ class Meta:
         fields = ()
 
 
-class GDACSEventFileterSet(filters.FilterSet):
-    countries = filters.ModelMultipleChoiceFilter(
-        field_name="countries",
-        queryset=Country.objects.all(),
-        widget=filters.widgets.CSVWidget,
-        method='filter_countries'
-    )
-    disaster_type = filters.NumberFilter(field_name="disaster_type", lookup_expr="exact")
-    publication_date__lte = filters.DateFilter(field_name="publication_date", lookup_expr="lte", input_formats=["%Y-%m-%d"])
-    publication_date__gte = filters.DateFilter(field_name="publication_date", lookup_expr="gte", input_formats=["%Y-%m-%d"])
-
-    class Meta:
-        model = GDACSEvent
-        fields = ()
-
-    def filter_countries(self, queryset, name, country):
-        if len(country):
-            return queryset.filter(countries__in=country).distinct()
-        return queryset
-
-
 class CountrySupportingPartnerFilter(filters.FilterSet):
     country = filters.ModelMultipleChoiceFilter(field_name="country", queryset=Country.objects.all())
 

api/filter_set.py

@@ -0,0 +1,28 @@
+import pandas as pd
+
+from django.core.management.base import BaseCommand
+
+from api.models import Country, District
+from api.logger import logger
+
+
+class Command(BaseCommand):
+    help = "Population Data for District To run, python manage.py ingest_admin1_population Population.GO.Admin1.Matched.xlsx"
+
+    def add_arguments(self, parser):
+        parser.add_argument("filename", nargs="+", type=str)
+
+    def handle(self, *args, **kwargs):
+        logger.info("Updating population data for District")
+        filename = kwargs["filename"][0]
+        district_df = pd.read_excel(filename, usecols=['GO Names', 'ISO3', '2016'])
+        district_df = district_df.rename(columns={'GO Names': 'District'})
+        count = 0
+        for index, row in district_df.iterrows():
+            district = District.objects.filter(name=row['District'].strip(), country__iso3=row['ISO3']).first()
+            if district:
+                count += 1
+                district.wb_population = row['2016']
+                district.wb_year = '2016' # Get this from excel file
+                district.save(update_fields=['wb_population', 'wb_year'])
+        logger.info(f'Updated population data for {count} district')

api/management/commands/ingest_admin1_population.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.24 on 2024-03-11 10:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0206_alter_export_export_type'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='export',
+            name='per_country',
+            field=models.IntegerField(blank=True, null=True, verbose_name='Per Country Id'),
+        ),
+        migrations.AlterField(
+            model_name='export',
+            name='export_type',
+            field=models.CharField(choices=[('dref-applications', 'Dref Applications'), ('dref-operational-updates', 'Dref Operational Updates'), ('dref-final-reports', 'Dref Final Reports'), ('per', 'Per')], max_length=255, verbose_name='Export Type'),
+        ),
+    ]

api/migrations/0207_auto_20240311_1044.py

@@ -2454,7 +2454,7 @@ class ExportType(models.TextChoices):
         DREF = 'dref-applications', _('Dref Applications')
         OPS_UPDATE = 'dref-operational-updates', _('Dref Operational Updates')
         FINAL_REPORT = 'dref-final-reports', _('Dref Final Reports')
-        PER = 'per-process', _('Per Process')
+        PER = 'per', _('Per')
 
     export_id = models.IntegerField(verbose_name=_('Export Id'))
     export_type = models.CharField(
@@ -2492,6 +2492,11 @@ class ExportType(models.TextChoices):
         null=True, blank=True,
         on_delete=models.SET_NULL,
     )
+    per_country = models.IntegerField(
+        verbose_name=_('Per Country Id'),
+        null=True,
+        blank=True
+    )
 
     def __str__(self):
         return f'{self.url} - {self.token}'

api/models.py

@@ -55,7 +55,6 @@
     CountryOfFieldReportToReview,
     Export,
     UserCountry,
-    GDACSEvent,
     CountryDirectory,
     CountryKeyDocument,
     NSDInitiatives,
@@ -2358,6 +2357,7 @@ class Meta:
     def create(self, validated_data):
         export_id = validated_data.get('export_id')
         export_type = validated_data.get('export_type')
+        country_id = validated_data.get('per_country')
         if export_type == Export.ExportType.DREF:
             title = Dref.objects.filter(
                 id=export_id
@@ -2378,7 +2378,10 @@ def create(self, validated_data):
         else:
             title = "Export"
         user = self.context['request'].user
-        validated_data['url'] = f'https://{settings.FRONTEND_URL}/{export_type}/{export_id}/export/'
+        if export_type == Export.ExportType.PER:
+            validated_data['url'] = f'https://{settings.FRONTEND_URL}/countries/{country_id}/{export_type}/{export_id}/export/'
+        else:
+            validated_data['url'] = f'https://{settings.FRONTEND_URL}/{export_type}/{export_id}/export/'
         validated_data['requested_by'] = user
         export = super().create(validated_data)
         if export.url:
@@ -2395,24 +2398,6 @@ def update(self, instance, validated_data):
         raise serializers.ValidationError("Update is not allowed")
 
 
-class GDACSEventSerializer(serializers.ModelSerializer):
-    population_value = serializers.IntegerField()
-    countries = MiniCountrySerializer(many=True)
-    disaster_type = DisasterTypeSerializer()
-    class Meta:
-        model = GDACSEvent
-        fields = (
-            "id",
-            "eventid",
-            "title",
-            "countries",
-            "population_unit",
-            "population_value",
-            "disaster_type",
-            "publication_date"
-        )
-
-
 class CountrySupportingPartnerSerializer(serializers.ModelSerializer):
     supporting_type_display = serializers.CharField(
         source="get_supporting_type_display",

api/serializers.py

@@ -13,66 +13,42 @@
 class Command(BaseCommand):
     help = 'Add minimum, maximum and Average temperature of country temperature data from source api'
 
-    def add_arguments(self, parser):
-        parser.add_argument(
-            '--start_date',
-            action='store',
-            help='Start date to fetch temperature data in the format YYYY-MM-DD'
-        )
-        parser.add_argument(
-            '--end_date',
-            action='store',
-            help='End date to fetch temperature data in the format YYYY-MM-DD'
-        )
 
     def handle(self, *args, **options):
-        for co in CountryOverview.objects.filter(country__record_type=CountryType.COUNTRY).all():
-            centroid = getattr(co.country, 'centroid', None)
-            if centroid:
-                centroid = json.loads(centroid.geojson)
-
-                lat = centroid.get("coordinates", [])[1]
-                lon = centroid.get("coordinates", [])[0]
-                if options['start_date'] and options['end_date']:
-                    response = requests.get(f'https://archive-api.open-meteo.com/v1/archive?latitude={lat}&longitude={lon}&start_date={options["start_date"]}&end_date={options["end_date"]}&daily=temperature_2m_max,temperature_2m_min,temperature_2m_mean,precipitation_sum,precipitation_hours')
-                else:
-                    response = requests.get(f'https://archive-api.open-meteo.com/v1/archive?latitude={lat}&longitude={lon}&start_date=2023-01-01&end_date=2023-12-31&daily=temperature_2m_max,temperature_2m_min,temperature_2m_mean,precipitation_sum,precipitation_hours')
-                if response.status_code != 200:
-                    continue
+        for co in CountryOverview.objects.filter(country__record_type=CountryType.COUNTRY, country__iso3__isnull=False).all():
+            country_iso3 = co.country.iso3
+            if country_iso3:
+                response = requests.get(f'https://climateknowledgeportal.worldbank.org/api/v1/cru-x0.5_climatology_tasmin,tas,tasmax,pr_climatology_monthly_1991-2020_mean_historical_cru_ts4.07_mean/{country_iso3}?_format=json')
                 response.raise_for_status()
-                try:
-                    data = response.json()
-                    daily_data = data.get('daily', {})
-                    if daily_data:
-                        monthly_temperatures = defaultdict(list)
-
-                        for i, date_str in enumerate(daily_data.get('time', [])):
-                            max_temp = daily_data.get('temperature_2m_max')[i]
-                            min_temp = daily_data.get('temperature_2m_min')[i]
-                            precipitation = daily_data.get('precipitation_sum')[i]
-                            if date_str and max_temp is not None and min_temp is not None:
-                                date = datetime.strptime(date_str, '%Y-%m-%d')
-                                month_key = (date.year, date.month)
-                                monthly_temperatures[month_key].append((max_temp, min_temp, precipitation))
-                        # Calculate min, max, and avg temperatures for each month
-                        for month_key, temps in monthly_temperatures.items():
-                            year, month = month_key
-                            max_temp_monthly = max(temps, key=lambda x: x[0])[0]
-                            min_temp_monthly = min(temps, key=lambda x: x[1])[1]
-                            avg_temp_monthly = (max_temp_monthly + min_temp_monthly) / 2
-                            precipitation_monthly = sum([x[2] for x in temps])
-
+            try:
+                response_data = response.json()
+                data = response_data.get('data', {})
+                if data:
+                    precipation = data.get('pr', {})
+                    average_temp = data.get('tas', {})
+                    min_temp = data.get('tasmin', {})
+                    max_temp = data.get('tasmax', {})
+                    merged_data = {
+                        country: {
+                            date: (precipation[country][date], average_temp[country][date], min_temp[country][date], max_temp[country][date])
+                            for date in precipation[country]
+                        } for country in precipation
+                    }
+                    for key , value in merged_data.items():
+                        for k, v in value.items():
+                            year_month = k.split('-')
+                            data = {
+                                'year': year_month[0],
+                                'month': year_month[1],
+                                'max_temp' : v[3],
+                                'min_temp' : v[2],
+                                'avg_temp': v[1],
+                                'precipitation' :v[0]
+                            }
                             CountryKeyClimate.objects.create(
                                 overview=co,
-                                year=year,
-                                month=month,
-                                max_temp=max_temp_monthly,
-                                min_temp=min_temp_monthly,
-                                avg_temp=avg_temp_monthly,
-                                precipitation=precipitation_monthly
+                                **data
                             )
-                        logger.info('Minimum, maximum and Average temperature of country temperature data added successfully')
-
-                except Exception as ex:
-                    logger.error(f'Error in ingesting climate data: {ex}')
-                    continue
+            except Exception as ex:
+                logger.error(f'Error in ingesting climate data',exc_info=True)
+                continue

databank/management/commands/ingest_climate.py

@@ -52,7 +52,6 @@ def handle(self, *args, **kwargs):
                         'page': page,
                         })
                     except requests.exceptions.HTTPError as err:
-                        print(err.response.text)
                         continue
                     try:
                         data_list = response.json()[1]
@@ -94,8 +93,8 @@ def handle(self, *args, **kwargs):
                 overview.world_bank_gni = indicators[5][0]
                 overview.world_bank_gender_inequality_index = indicators[6][0]
                 overview.world_bank_life_expectancy = indicators[7][0]
-                overview.world_bank_literacy_rate = indicators[0][0]
-                overview.world_bank_poverty_rate = indicators[1][0]
+                overview.world_bank_literacy_rate = indicators[8][0]
+                overview.world_bank_poverty_rate = indicators[9][0]
                 overview.save(
                     update_fields=[
                         'world_bank_population',

databank/management/commands/ingest_worldbank.py

@@ -119,8 +119,7 @@ cronjobs:
     schedule: '1 0 * * *'
   - command: 'update_surge_alert_status'
     schedule: '1 */12 * * *'
-  - command: 'ingest_gdacs'
-    schedule: '1 */12 * * *'
+
 
 elasticsearch:
   enabled: true