Merge pull request #27 from wmo-raf/dev

Updates and Bug Fixes

Author: erick-otenyo

forecastmanager/forecast_settings.py

@@ -62,7 +62,7 @@ def periods_as_choices(self):
     @property
     def effective_periods(self):
         return [
-            {"label": period.label, "time": period.forecast_effective_time, "default": period.default}
+            {"label": period.label, "time": period.forecast_effective_time}
             for period in self.periods.all()]
 
     @property
@@ -73,27 +73,20 @@ def weather_conditions_list(self):
 
 class ForecastPeriod(Orderable):
     parent = ParentalKey(ForecastSetting, on_delete=models.CASCADE, related_name="periods")
-    default = models.BooleanField(default=False, verbose_name=_("Is default"))
-    forecast_effective_time = models.TimeField(verbose_name=_("Forecast Effective Time"))
+    forecast_effective_time = models.TimeField(verbose_name=_("Forecast Effective Time"), unique=True)
     label = models.CharField(max_length=100, verbose_name=_("Label"))
 
     class Meta:
-        unique_together = ("default", "forecast_effective_time")
+        ordering = ["forecast_effective_time"]
 
     panels = [
         FieldPanel('forecast_effective_time'),
         FieldPanel('label'),
-        FieldPanel('default'),
     ]
 
     def __str__(self):
         return self.label
 
-    def save(self, *args, **kwargs):
-        if self.default:
-            ForecastPeriod.objects.filter(default=True).update(default=False)
-        super().save(*args, **kwargs)
-
 
 class ForecastDataParameters(Orderable):
     PARAMETER_TYPE_CHOICES = (
@@ -136,7 +129,8 @@ def parameter_info(self):
         return WEATHER_PARAMETERS_AS_DICT.get(self.parameter)
 
     def parse_value(self, value):
-        # TODO: Implement parsing for different parameter types
+        if self.parameter_type == "numeric":
+            return float(value)
         return value
 
 

forecastmanager/forms.py

@@ -97,7 +97,7 @@ def clean(self):
 
             # check parameters
             for param, value in params_data.items():
-                if value:
+                if value is not None and value != "":
                     param = ForecastDataParameters.objects.filter(name=param).first()
                     if not param:
                         self.add_error(None, f"Unknown parameter found in table data: {param}")

forecastmanager/management/commands/clear_old_forecasts.py

@@ -0,0 +1,30 @@
+import logging
+
+from django.core.management.base import BaseCommand
+from django.utils import timezone
+
+from forecastmanager.models import Forecast
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = 'Clear old forecasts from the database.'
+
+    def handle(self, *args, **options):
+        logger.info("Clearing old forecasts from the database...")
+
+        current_time = timezone.localtime()
+
+        # Get all forecasts that are older than the current time
+        old_forecasts = Forecast.objects.filter(forecast_date__lt=current_time)
+
+        if not old_forecasts:
+            logger.info("No old forecasts found.")
+            return
+
+        # Delete the old forecasts
+        logger.info(f"Deleting {old_forecasts.count()} old forecasts...")
+        old_forecasts.delete()
+
+        logger.info("Old forecasts deleted successfully.")

forecastmanager/management/commands/generate_forecast.py

@@ -3,24 +3,50 @@
 import requests
 from dateutil.parser import parse
 from django.core.management.base import BaseCommand
+from django.utils import timezone
 from wagtail.models import Site
 
-from forecastmanager.forecast_settings import ForecastSetting, WeatherCondition, ForecastPeriod, ForecastDataParameters
-from forecastmanager.models import City, CityForecast, DataValue, Forecast
 from forecastmanager.constants import WEATHER_CONDITIONS_AS_DICT
+from forecastmanager.forecast_settings import (
+    ForecastSetting,
+    WeatherCondition,
+    ForecastPeriod,
+    ForecastDataParameters
+)
+from forecastmanager.models import (
+    City,
+    CityForecast,
+    DataValue,
+    Forecast
+)
 
 logger = logging.getLogger(__name__)
 
 # Define the base URL for the Met Norway API
-BASE_URL = "https://www.yr.no/api/v0/locations"
+BASE_URL = "https://api.met.no/weatherapi/locationforecast/2.0/complete"
+
+DEFAULT_INSTANT_DATA_PARAMETERS = [
+    {"parameter": "air_pressure_at_sea_level", "name": "Air Pressure (Sea level)", "parameter_unit": "hPa"},
+    {"parameter": "air_temperature", "name": "Minimum Air Temperature", "parameter_unit": "°C"},
+    {"parameter": "wind_speed", "name": "Wind Speed", "parameter_unit": "m/s"},
+    {"parameter": "wind_from_direction", "name": "Wind Direction ", "parameter_unit": "degrees"}
+]
+
+DEFAULT_NEXT_HOURS_DATA_PARAMETERS = [
+    {"parameter": "air_temperature_min", "name": "Minimum Air Temperature", "parameter_unit": "°C"},
+    {"parameter": "air_temperature_max", "name": "Maximum Air Temperature", "parameter_unit": "°C"},
+    {"parameter": "precipitation_amount", "name": "Precipitation Amount", "parameter_unit": "mm"},
+]
+
+DEFAULT_PARAMETERS = DEFAULT_INSTANT_DATA_PARAMETERS + DEFAULT_NEXT_HOURS_DATA_PARAMETERS
 
 
 class Command(BaseCommand):
     help = ('Get the weather forecast for the next 7 days from Yr.no '
             'for all cities in the database and save it to the database.')
 
     def handle(self, *args, **options):
-        print("Getting 7 Day Forecast from Yr.no...")
+        logger.info("Getting 7 Day Forecast from Yr.no...")
 
         cities = City.objects.all()
         if not cities:
@@ -35,7 +61,13 @@ def handle(self, *args, **options):
 
         forecast_setting = ForecastSetting.for_site(site)
 
-        user_agent = f"{site.site_name} (WMO NMHSs Website Template) {site.root_url}"
+        site_name = site.site_name
+        root_url = site.root_url
+
+        user_agent = f"ClimWeb {root_url}"
+        if site_name:
+            user_agent = f"{site_name}/{user_agent}"
+
         user_agent = user_agent.strip()
 
         if not forecast_setting.enable_auto_forecast:
@@ -46,83 +78,60 @@ def handle(self, *args, **options):
         conditions_by_symbol = {condition.symbol: condition for condition in conditions}
 
         parameters = forecast_setting.data_parameters.all()
+
         if not parameters.exists():
             # create default forecast parameters
-            default_parameters = [
-                {"parameter": "air_temperature_max", "name": "Maximum Air Temperature", "parameter_unit": "°C"},
-                {"parameter": "air_temperature_min", "name": "Minimum Air Temperature", "parameter_unit": "°C"},
-                {"parameter": "wind_speed", "name": "Wind Speed", "parameter_unit": "m/s"},
-                {"parameter": "precipitation_amount", "name": "Precipitation Amount", "parameter_unit": "mm"}
-            ]
-
-            for default_parameter in default_parameters:
+            for default_parameter in DEFAULT_PARAMETERS:
                 ForecastDataParameters.objects.create(parent=forecast_setting, **default_parameter)
 
             parameters = forecast_setting.data_parameters.all()
 
         parameters_dict = {parameter.parameter: parameter for parameter in parameters}
 
-        forecast_periods = forecast_setting.periods.all()
-        if not forecast_periods.exists():
-            # create default forecast period
-            ForecastPeriod.objects.create(parent=forecast_setting,
-                                          label="Whole Day",
-                                          forecast_effective_time="00:00:00",
-                                          default=True)
-
-        forecast_period = forecast_periods.filter(default=True).first()
-        if not forecast_period:
-            # pick first one if no default
-            forecast_period = forecast_periods.first()
-
         cities_data = {}
 
         for city in cities:
-            print(f"Getting forecast for {city.name}...")
+            logger.info(f"Getting forecast for {city.name}...")
 
             lon = city.x
             lat = city.y
 
-            url = f"{BASE_URL}/{lat},{lon}/forecast"
+            url = f"{BASE_URL}?lat={lat}&lon={lon}"
 
             # Send a GET request to the API
             response = requests.get(url, headers={"User-Agent": user_agent})
 
             if response.status_code >= 400:
-                logger.error(
-                    f"Failed to get forecast for {city.name}. Status code: {response.status_code}")
+                logger.error(f"Failed to get forecast for {city.name}. Status code: {response.status_code}")
                 continue
 
             # Get the weather data from the response
             data = response.json()
 
-            day_intervals = data['dayIntervals']
+            # Get the timeseries data from the response
+            timeseries = data.get('properties', {}).get('timeseries')
 
-            for day in day_intervals[:8]:
-                date = parse(day.get("start"))
-                condition = day.get("twentyFourHourSymbol")
-                temperature_data = day.get("temperature")
-                air_temperature_max = temperature_data.get("max")
-                air_temperature_min = temperature_data.get("min")
+            # Get the first and last datetime for the forecast
+            first_datetime = timezone.localtime().replace(hour=0, minute=0, second=0, microsecond=0)
+            max_days = 6
+            last_datetime = (first_datetime + timezone.timedelta(days=max_days)).replace(hour=23, minute=59, second=59)
 
-                wind_data = day.get("wind")
-                wind_speed = wind_data.get("max")
+            # Create a forecast for the city
+            for time_data in timeseries:
+                time = time_data.get("time")
+                utc_date = parse(time)
+                timezone_date = timezone.localtime(utc_date)
 
-                precipitation_data = day.get("precipitation")
-                precipitation = precipitation_data.get("value")
+                # Check if the forecast is within the next 7 days
+                if timezone_date < first_datetime or timezone_date > last_datetime:
+                    continue
 
-                data_values = {
-                    "date": date,
-                    "condition": condition,
-                    "parameters": {
-                        "air_temperature_max": air_temperature_max,
-                        "air_temperature_min": air_temperature_min,
-                        "wind_speed": wind_speed,
-                        "precipitation_amount": precipitation
-                    }
-                }
+                data_values = time_data.get("data", {})
 
-                condition = data_values.get('condition')
+                # Get the weather condition for the forecast
+                condition = data_values.get("next_1_hours", {}).get("summary", {}).get("symbol_code")
+                if condition is None:
+                    condition = data_values.get("next_6_hours", {}).get("summary", {}).get("symbol_code")
 
                 if conditions_by_symbol.get(condition) is None:
                     condition_info = WEATHER_CONDITIONS_AS_DICT.get(condition)
@@ -142,25 +151,58 @@ def handle(self, *args, **options):
                     continue
 
                 city_forecast = CityForecast(city=city, condition=condition_obj)
-                for key, value in data_values.get("parameters", {}).items():
+
+                instant_data = data_values.get("instant", {}).get("details", {})
+                # Add the instant data values to the forecast
+                for key, value in instant_data.items():
                     if parameters_dict.get(key) is None:
                         continue
 
                     parameter = parameters_dict[key]
                     data_value = DataValue(parameter=parameter, value=value)
                     city_forecast.data_values.add(data_value)
 
-                if date in cities_data:
-                    cities_data[date].append(city_forecast)
+                # Add the next hours data values to the forecast
+                for param in DEFAULT_NEXT_HOURS_DATA_PARAMETERS:
+                    param_key = param.get("parameter")
+                    if parameters_dict.get(param_key) is None:
+                        continue
+
+                    next_1_hours_data = data_values.get("next_1_hours", {}).get("details", {})
+                    next_6_hours_data = data_values.get("next_6_hours", {}).get("details", {})
+
+                    next_data_value = None
+                    if param_key in next_1_hours_data:
+                        next_data_value = DataValue(parameter=parameters_dict[param_key],
+                                                    value=next_1_hours_data[param_key])
+                    elif param_key in next_6_hours_data:
+                        next_data_value = DataValue(parameter=parameters_dict[param_key],
+                                                    value=next_6_hours_data[param_key])
+
+                    if next_data_value is not None:
+                        city_forecast.data_values.add(next_data_value)
+
+                # Add the forecast to the cities data
+                if timezone_date in cities_data:
+                    cities_data[timezone_date].append(city_forecast)
                 else:
-                    cities_data[date] = [city_forecast]
+                    cities_data[timezone_date] = [city_forecast]
+
+        # Create the forecast for the cities
+        for forecast_date, city_forecasts in cities_data.items():
+            effective_time = f"{forecast_date.hour}:00"
+
+            forecast_period = ForecastPeriod.objects.filter(forecast_effective_time=effective_time).first()
+            if forecast_period is None:
+                forecast_period = ForecastPeriod.objects.create(parent=forecast_setting,
+                                                                forecast_effective_time=effective_time,
+                                                                label=effective_time)
 
-        for time, city_forecasts in cities_data.items():
-            forecast = Forecast.objects.filter(forecast_date=time, effective_period=forecast_period)
+            forecast = Forecast.objects.filter(forecast_date=forecast_date, effective_period=forecast_period)
             if forecast.exists():
                 forecast.delete()
 
-            forecast = Forecast(forecast_date=time, effective_period=forecast_period, source="yr")
+            forecast = Forecast(forecast_date=forecast_date, effective_period=forecast_period, source="yr")
 
             for city_forecast in city_forecasts:
                 forecast.city_forecasts.add(city_forecast)

forecastmanager/migrations/0025_alter_forecast_options_alter_forecastperiod_options.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.2.7 on 2024-06-06 06:37
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('forecastmanager', '0024_alter_forecast_options_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='forecast',
+            options={'ordering': ['forecast_date', 'effective_period'], 'verbose_name': 'Forecast', 'verbose_name_plural': 'Forecasts'},
+        ),
+        migrations.AlterModelOptions(
+            name='forecastperiod',
+            options={'ordering': ['forecast_effective_time']},
+        ),
+    ]

forecastmanager/migrations/0026_alter_forecastperiod_unique_together_and_more.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.2.3 on 2024-06-13 09:02
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('forecastmanager', '0025_alter_forecast_options_alter_forecastperiod_options'),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name='forecastperiod',
+            unique_together=set(),
+        ),
+        migrations.AlterField(
+            model_name='forecastperiod',
+            name='forecast_effective_time',
+            field=models.TimeField(unique=True, verbose_name='Forecast Effective Time'),
+        ),
+        migrations.RemoveField(
+            model_name='forecastperiod',
+            name='default',
+        ),
+    ]