diff --git a/pipelines/assets/fixtures.py b/pipelines/assets/fixtures.py index 8beb7e6..431335c 100644 --- a/pipelines/assets/fixtures.py +++ b/pipelines/assets/fixtures.py @@ -110,6 +110,15 @@ def validate_instances( + instance["livelihood_strategy"][2:] # strategy_type, season, product_id, additional_identifier + [instance["wealth_group"][3]] # full_name ) + elif model_name == "WealthGroupCharacteristicValue": + instance["natural_key"] = instance["wealth_group"][ + :3 # livelihood_zone, reference_year_end_date, wealth_group_category + ] + [ + instance["wealth_characteristic_id"], + instance["reference_type"], + instance["product_id"] or "", # Natural key components must be "" rather than None + instance["wealth_group"][3], # full_name + ] # The natural key is a list of strings, or possibly numbers, so validate that here to avoid confusing # error messages later. if "natural_key" not in instance: @@ -145,7 +154,7 @@ def validate_instances( if column not in instance: error = f"Missing mandatory field {field.name} for {record_reference}" model_errors.append(error) - elif not instance[column]: + elif not instance[column] and not instance[column] == 0: error = f"Missing value for mandatory field {column} for {record_reference}" model_errors.append(error) # Ensure the instances contain valid parent references for foreign keys @@ -419,13 +428,8 @@ def imported_communities( """ Communities from a BSS, imported to the Django database using a fixture. """ - fixture, metadata = get_fixture_from_instances(community_instances) - metadata = import_fixture(fixture) - - return Output( - None, - metadata=metadata, - ) + output = get_fixture_from_instances(community_instances) + return import_fixture(output.value) @asset(partitions_def=bss_instances_partitions_def) diff --git a/pipelines/assets/wealth_characteristic.py b/pipelines/assets/wealth_characteristic.py index 0b6b6b7..034e2a0 100644 --- a/pipelines/assets/wealth_characteristic.py +++ b/pipelines/assets/wealth_characteristic.py @@ -531,16 +531,7 @@ def wealth_characteristic_valid_instances( Valid WealthGroup and WealthGroupCharacteristicValue instances from a BSS, ready to be loaded via a Django fixture. """ partition_key = context.asset_partition_key_for_output() - valid_instances, metadata = validate_instances(context, config, wealth_characteristic_instances, partition_key) - metadata = {f"num_{key.lower()}": len(value) for key, value in valid_instances.items()} - metadata["total_instances"] = sum(len(value) for value in valid_instances.values()) - metadata["preview"] = MetadataValue.md( - f"```json\n{json.dumps(valid_instances, indent=4, ensure_ascii=False)}\n```" - ) - return Output( - valid_instances, - metadata=metadata, - ) + return validate_instances(context, config, wealth_characteristic_instances, partition_key) @asset(partitions_def=bss_instances_partitions_def, io_manager_key="json_io_manager")