Merge branch 'main' into HEA-596/Relax-mandatory-field-requirements-for-some-LivelihoodActivity-subclasses

Author: girumb

pipelines/assets/base.py

@@ -190,7 +190,7 @@ def corrected_files(context: AssetExecutionContext, config: BSSMetadataConfig) -
     partition_key = context.asset_partition_key_for_output()
     livelihood_zone_baseline = LivelihoodZoneBaseline.objects.get_by_natural_key(*partition_key.split("~")[1:])
 
-    def validate_previous_value(cell, expected_previous_value, previous_value):
+    def validate_previous_value(worksheet_name, cell_reference, expected_previous_value, previous_value):
         """
         Inline function to validate the existing value of a cell is the expected one, prior to correcting it.
         """
@@ -210,7 +210,7 @@ def validate_previous_value(cell, expected_previous_value, previous_value):
         if expected_previous_value != previous_value:
             raise ValueError(
                 "Unexpected prior value in source BSS. "
-                f"BSS `{partition_key}`, cell `{cell}`, "
+                f"BSS `{partition_key}`, cell `'{worksheet_name}'!{cell_reference}`, "
                 f"value found `{previous_value}`, expected `{expected_previous_value}`."
             )
 
@@ -280,12 +280,16 @@ def validate_previous_value(cell, expected_previous_value, previous_value):
                             previous_value = xlrd.error_text_from_code[
                                 xlrd_wb.sheet_by_name(correction.worksheet_name).cell_value(row - 1, col - 1)
                             ]
-                        validate_previous_value(cell, correction.previous_value, previous_value)
+                        validate_previous_value(
+                            correction.worksheet_name, cell, correction.previous_value, previous_value
+                        )
                         # xlwt uses 0-based indexes, but coordinate_to_tuple uses 1-based, so offset the values
                         wb.get_sheet(correction.worksheet_name).write(row - 1, col - 1, correction.value)
                     else:
                         cell = wb[correction.worksheet_name][cell]
-                        validate_previous_value(cell, correction.previous_value, cell.value)
+                        validate_previous_value(
+                            correction.worksheet_name, cell.coordinate, correction.previous_value, cell.value
+                        )
                         cell.value = correction.value
                         cell.comment = Comment(
                             f"{correction.author__username} on {correction.correction_date.date().isoformat()}: {correction.comment}",  # NOQA: E501

pipelines/assets/livelihood_activity_regexes.json

@@ -739,6 +739,12 @@
         false,
         "number_of_local_measures"
     ],
+    [
+        "autre culture(?: de rente)?{separator_pattern} (?:type|nom)",
+        null,
+        true,
+        "product__name"
+    ],
     [
         "autre culture{separator_pattern} \\(?{product_pattern}\\)?(?: type)?",
         null,
@@ -769,12 +775,6 @@
         true,
         "quantity_sold"
     ],
-    [
-        "autre culture de rente: type",
-        null,
-        true,
-        "product__name"
-    ],
     [
         "autre culture de rente: \\(?{product_pattern}\\)?",
         null,
@@ -793,6 +793,12 @@
         true,
         "quantity_sold"
     ],
+    [
+        "{product_pattern}: (?P<unit_of_measure_id>kg|l)? {vendu_pattern}",
+        null,
+        true,
+        "quantity_sold"
+    ],
     [
         "{product_pattern}: (?P<unit_of_measure_id>kg|l)? sold",
         null,

pipelines_tests/test_assets/test_livelihood_activity_regexes.json

@@ -519,6 +519,10 @@
         "is_start": true,
         "attribute": "product__name"
     },
+    "Autre culture: nom": {
+        "is_start": true,
+        "attribute": "product__name"
+    },
     "chou: qtité produite": {
         "is_start": true,
         "product_id": "chou",