Merge pull request #131 from FEWS-NET/HEA-572/relax_data_completeness_constraint

HEA-572 Add field-level checks to xxx_valid_instances

Author: girumb

apps/baseline/migrations/0017_alter_livelihoodactivity_percentage_kcals.py

@@ -0,0 +1,25 @@
+# Generated by Django 5.1.1 on 2024-11-22 03:51
+
+import django.core.validators
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("baseline", "0016_alter_livelihoodstrategy_additional_identifier_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="livelihoodactivity",
+            name="percentage_kcals",
+            field=models.FloatField(
+                blank=True,
+                help_text="Percentage of annual household kcal requirement provided by this livelihood strategy",
+                null=True,
+                validators=[django.core.validators.MinValueValidator(0)],
+                verbose_name="Percentage of required kcals",
+            ),
+        ),
+    ]

apps/baseline/models.py

@@ -1125,6 +1125,7 @@ class LivelihoodActivity(common_models.Model):
     percentage_kcals = models.FloatField(
         blank=True,
         null=True,
+        validators=[MinValueValidator(0)],
         verbose_name=_("Percentage of required kcals"),
         help_text=_("Percentage of annual household kcal requirement provided by this livelihood strategy"),
     )

pipelines/assets/fixtures.py

@@ -6,6 +6,7 @@
 from io import StringIO
 
 import django
+import numpy as np
 import pandas as pd
 from dagster import AssetExecutionContext, MetadataValue, Output, asset
 from django.core.files import File
@@ -174,6 +175,31 @@ def validate_instances(
                         )
                         errors.append(error)
 
+            # Use the Django model to validate the fields, so we can apply already defined model validations and
+            # return informative error messages.
+            fields = [
+                field
+                for field in model._meta.concrete_fields
+                if not isinstance(field, models.ForeignKey) and field.name in df
+            ]
+            instance = model()
+            for record in df.replace(np.nan, None).itertuples():
+                for field in fields:
+                    value = getattr(record, field.name)
+                    if not value and field.null:
+                        # Replace empty strings with None for optional fields
+                        value = None
+                    try:
+                        field.clean(value, instance)
+                    except Exception as e:
+                        error = (
+                            f'Invalid {field.name} value {value}:  "{", ".join(e.error_list[0].messages)}"\nRecord '
+                            f"{record.Index} from cell '{record.bss_sheet}'!{record.bss_column}{record.bss_row} "
+                            f"for {model_name} in record "
+                            f'{str({k: v for k,v in record._asdict().items() if k != "Index"})}.'
+                        )
+                        errors.append(error)
+
             # Check that the kcals/kg matches the values in the ClassifiedProduct model, if it's present in the BSS
             if model_name == "LivelihoodActivity" and "product__kcals_per_unit" in df:
                 df["product"] = df["livelihood_strategy"].apply(lambda x: x[4])
@@ -182,7 +208,7 @@ def validate_instances(
                 df["reference_unit_of_measure"] = df["product"].apply(lambda x: x.unit_of_measure)
                 for record in df[df["product__kcals_per_unit"] != df["reference_kcals_per_unit"]].itertuples():
                     error = (
-                        f"Non-standard value {record.product__kcals_per_unit} in '{record.column}"
+                        f"Non-standard value {record.product__kcals_per_unit} in '{record.column}' "
                         f"for {model_name} in record "
                         f'{str({k: v for k,v in record._asdict().items() if k != "Index"})}. '
                         f"Expected {record.reference_kcals_per_unit}/{record.reference_unit_of_measure} for {record.product}"

pipelines/assets/livelihood_activity.py

@@ -180,7 +180,7 @@ def get_livelihood_activity_regexes() -> list:
     placeholder_patterns = {
         "label_pattern": r"[a-zà-ÿ][a-zà-ÿ',/ \.\>\-\(\)]+?",
         "product_pattern": r"(?P<product_id>[a-zà-ÿ][a-zà-ÿ',/ \.\>\-\(\)]+?)",
-        "season_pattern": r"(?P<season>season [12]|saison [12]|[12][a-z] season||[12][a-zà-ÿ] saison|r[eé]colte principale|gu|deyr+?)",  # NOQA: E501
+        "season_pattern": r"(?P<season>season [12]|saison [12]|[12][a-z] season||[12][a-zà-ÿ] saison|r[eé]colte principale|principale r[eé]colte|gu|deyr+?)",  # NOQA: E501
         "additional_identifier_pattern": r"\(?(?P<additional_identifier>rainfed|irrigated|pluviale?|irriguée|submersion libre|submersion contrôlée|flottant)\)?",
         "unit_of_measure_pattern": r"(?P<unit_of_measure_id>[a-z]+)",
         "nbr_pattern": r"(?:n[b|o]r?)\.?",