Hea 572/relax data completeness constraint

pipelines/__init__.py

@@ -10,40 +10,50 @@
 )
 from .assets.baseline import baseline_instances, community_instances
 from .assets.fixtures import (
-    consolidated_fixtures,
-    consolidated_instances,
-    imported_baselines,
+    consolidated_fixture,
+    imported_baseline,
     imported_communities,
     uploaded_baselines,
-    validated_instances,
 )
 from .assets.livelihood_activity import (
     all_livelihood_activity_labels_dataframe,
+    imported_livelihood_activities,
     livelihood_activity_dataframe,
+    livelihood_activity_fixture,
     livelihood_activity_instances,
     livelihood_activity_label_dataframe,
+    livelihood_activity_valid_instances,
     summary_livelihood_activity_labels_dataframe,
 )
 from .assets.other_cash_income import (
     all_other_cash_income_labels_dataframe,
+    imported_other_cash_income_activities,
     other_cash_income_dataframe,
+    other_cash_income_fixture,
     other_cash_income_instances,
     other_cash_income_label_dataframe,
+    other_cash_income_valid_instances,
     summary_other_cash_income_labels_dataframe,
 )
 from .assets.wealth_characteristic import (
     all_wealth_characteristic_labels_dataframe,
+    imported_wealth_characteristics,
     summary_wealth_characteristic_labels_dataframe,
     wealth_characteristic_dataframe,
+    wealth_characteristic_fixture,
     wealth_characteristic_instances,
     wealth_characteristic_label_dataframe,
+    wealth_characteristic_valid_instances,
 )
 from .assets.wild_foods import (
     all_wild_foods_labels_dataframe,
+    imported_wild_foods_activities,
     summary_wild_foods_labels_dataframe,
     wild_foods_dataframe,
+    wild_foods_fixture,
     wild_foods_instances,
     wild_foods_label_dataframe,
+    wild_foods_valid_instances,
 )
 from .jobs.fixtures import (
     extract_dataframes,
@@ -76,27 +86,37 @@
         all_livelihood_activity_labels_dataframe,
         summary_livelihood_activity_labels_dataframe,
         livelihood_activity_instances,
+        livelihood_activity_valid_instances,
+        livelihood_activity_fixture,
+        imported_livelihood_activities,
         other_cash_income_dataframe,
         other_cash_income_label_dataframe,
         all_other_cash_income_labels_dataframe,
         summary_other_cash_income_labels_dataframe,
         other_cash_income_instances,
+        other_cash_income_valid_instances,
+        other_cash_income_fixture,
+        imported_other_cash_income_activities,
         wild_foods_dataframe,
         wild_foods_label_dataframe,
         all_wild_foods_labels_dataframe,
         summary_wild_foods_labels_dataframe,
         wild_foods_instances,
+        wild_foods_valid_instances,
+        wild_foods_fixture,
+        imported_wild_foods_activities,
         wealth_characteristic_dataframe,
         wealth_characteristic_label_dataframe,
         all_wealth_characteristic_labels_dataframe,
         wealth_characteristic_instances,
+        wealth_characteristic_valid_instances,
+        wealth_characteristic_fixture,
+        imported_wealth_characteristics,
         summary_wealth_characteristic_labels_dataframe,
-        consolidated_instances,
-        validated_instances,
-        consolidated_fixtures,
+        consolidated_fixture,
         uploaded_baselines,
         imported_communities,
-        imported_baselines,
+        imported_baseline,
     ],
     jobs=[
         update_metadata,

pipelines/assets/baseline.py

@@ -83,6 +83,9 @@ def get_wealth_group_dataframe(
             wealth_group_df = wealthgroupcategorylookup.do_lookup(
                 wealth_group_df, "district", "wealth_group_category", update=True
             )
+            # Remove the duplicate wealth_group_category_original column created by the second do_lookup(),
+            # which otherwise causes problems when trying to merge dataframes, e.g. when building the wealth_group_df.
+            wealth_group_df = wealth_group_df.loc[:, ~wealth_group_df.columns.duplicated()]
         except ValueError:
             pass