Update Level 1b consumable availability to reflect 1b-2 merge (#1743)

* update availability of consumables at level 1b to reflect the weighted average of availability at levels 1b and 2

* allow for different weights when calculating average

* revert original resource file

* add new resource files with different ways of handling level 1b availability

* merge in xpert update from master

* apply weighted average of level 1b and 2 facilities to level 1b probability across alternative consumable scenarios

- set up so that `consumable_availability_estimation.py` produces the final RF using functions from `generate_consumable_availability_scenarios_for_impact_analysis.py`

* remove test ResourceFiles applying different level 1b-2 weights.

* update logical check for changes in consumables availabiltity in routine `update_consumables_availability_to_represent_merging_of_levels_1b_and_2`

* Add note

* linting

* introduce switch to allow use of the original consumables dataset

* correct filename

* check both consumables data files

* update old scripts to point to original consumbales file to preserve consistency.

* linting

---------

Co-authored-by: Tim Hallett <39991060+tbhallett@users.noreply.github.com>

Author: sakshimohan

resources/healthsystem/ResourceFile_HealthSystem_parameters.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ff3be5d065013bc7c5cffd982f04fa3b37a923ebd3f37e5dbb39542703dd2310
-size 914
+oid sha256:b96e35b518fb33fe0c89b9e47d6b623ec9eb30d454b404554312787a5937404e
+size 967

resources/healthsystem/consumables/ResourceFile_Consumables_availability_small.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3bc044868ad9f1c618fc9b3fd6fc0ed23f32b153c8438f52fd4b72e40d8f503f
-size 55831644
+oid sha256:2ab824924a4c0e798eeec6c2514c50577ba83185120ec257d5065c583fe1dfa6
+size 57341233

resources/healthsystem/consumables/ResourceFile_Consumables_availability_small_original.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3bc044868ad9f1c618fc9b3fd6fc0ed23f32b153c8438f52fd4b72e40d8f503f
+size 55831644

src/scripts/contraception/Items_InclAvailabilityData.py

@@ -27,7 +27,7 @@
 last_line_interv_pkg_name = 'Female Condom'
 name_containing = 'sutur'
 what_to_check = 'lines'  # 'lines' or 'item_name'
-avail_data_filename = 'ResourceFile_Consumables_availability_small.csv'
+avail_data_filename = 'ResourceFile_Consumables_availability_small_original.csv'
 #####
 
 avail_data = pd.read_csv(Path(

src/scripts/data_file_processing/healthsystem/consumables/consumable_resource_analyses_with_lmis/consumables_availability_estimation.py

@@ -15,7 +15,7 @@
 
 # get consumables spreadsheet
 cons_availability = pd.read_csv(
-    resourcefilepath / "healthsystem/consumables/ResourceFile_Consumables_availability_small.csv")
+    resourcefilepath / "healthsystem/consumables/ResourceFile_Consumables_availability_small_original.csv")
 items_list = pd.read_csv(
     resourcefilepath / "healthsystem/consumables/ResourceFile_Consumables_Items_and_Packages.csv")
 

src/scripts/data_file_processing/healthsystem/consumables/generating_consumable_scenarios/generate_consumable_availability_scenarios_for_impact_analysis.py

@@ -52,7 +52,8 @@
 #                                                                     those of '2' and have more overall capacity, so
 #                                                                     probably account for the majority of the
 #                                                                     interactions.
-
+# Note that, as of PR #1743, this should not be changed, as the availability of consumables at level 1b is now
+# encoded to reflect a (weighted) average of the availability of levels '1b' and '2'.
 
 def pool_capabilities_at_levels_1b_and_2(df_original: pd.DataFrame) -> pd.DataFrame:
     """Return a modified version of the imported capabilities DataFrame to reflect that the capabilities of level 1b
@@ -200,6 +201,12 @@ class HealthSystem(Module):
             Types.DATA_FRAME,
             "Look-up table for the designations of consumables (whether diagnostic, medicine, or other",
         ),
+        "data_source_for_cons_availability_estimates": Parameter(
+            Types.STRING, "Source of data on consumable availability. Options are: `original` or `updated`."
+                          "The original source was used in the calibration and presented in the overview paper. The "
+                          "updated source introduced in PR #1743 and better reflects the average availability of "
+                          "consumables in the merged 1b/2 facility level."
+        ),
         "availability_estimates": Parameter(
             Types.DATA_FRAME, "Estimated availability of consumables in the LMIS dataset."
         ),
@@ -655,10 +662,20 @@ def read_parameters(self, resourcefilepath: Optional[Path] = None):
             path_to_resourcefiles_for_healthsystem / "consumables" / "ResourceFile_Consumables_Item_Designations.csv",
             dtype={"Item_Code": int, "is_diagnostic": bool, "is_medicine": bool, "is_other": bool},
         ).set_index("Item_Code")
+
+        # Choose to read-in the updated availabilty estimates or the legacy availability estimates
+        if self.parameters["data_source_for_cons_availability_estimates"] == 'original':
+            filename_for_cons_availability_estimates = "ResourceFile_Consumables_availability_small_original.csv"
+        elif self.parameters["data_source_for_cons_availability_estimates"] == 'updated':
+            filename_for_cons_availability_estimates = "ResourceFile_Consumables_availability_small.csv"
+        else:
+            raise ValueError("data_source_for_cons_availability_estimates should be either 'original' or 'updated'")
+
         self.parameters["availability_estimates"] = pd.read_csv(
-            path_to_resourcefiles_for_healthsystem / "consumables" / "ResourceFile_Consumables_availability_small.csv"
+            path_to_resourcefiles_for_healthsystem / "consumables" / filename_for_cons_availability_estimates
         )
 
+
         # Data on the number of beds available of each type by facility_id
         self.parameters["BedCapacity"] = pd.read_csv(
             path_to_resourcefiles_for_healthsystem / "infrastructure_and_equipment" / "ResourceFile_Bed_Capacity.csv"
@@ -1330,7 +1347,7 @@ def update_consumables_availability_to_represent_merging_of_levels_1b_and_2(self
         assert (df_updated.columns == df_original.columns).all()
         assert (df_updated.dtypes == df_original.dtypes).all()
 
-        # check values the same for everything apart from the facility level '2' facilities
+        # check values the same for everything apart from the facility level 'LABEL_FOR_MERGED_FACILITY_LEVELS_1B_AND_2'
         facilities_with_any_differences = set(
             df_updated.loc[
                 ~(
@@ -1339,8 +1356,10 @@ def update_consumables_availability_to_represent_merging_of_levels_1b_and_2(self
                 "Facility_ID",
             ]
         )
-        level2_facilities = set(mfl.loc[mfl["Facility_Level"] == "2", "Facility_ID"])
-        assert facilities_with_any_differences.issubset(level2_facilities)
+        updated_facilities = set(
+            mfl.loc[mfl['Facility_Level'] == LABEL_FOR_MERGED_FACILITY_LEVELS_1B_AND_2, 'Facility_ID']
+        )
+        assert facilities_with_any_differences.issubset(updated_facilities)
 
         return df_updated
 

src/scripts/malaria/malaria_cons_availability.py

@@ -531,9 +531,13 @@ def schedule_hsi_that_will_request_consumables(sim):
 
 def test_check_format_of_consumables_file():
     """Run the check on the file used by default for the Consumables data"""
+    path_to_file = resourcefilepath / 'healthsystem' / 'consumables'
     check_format_of_consumables_file(
-        pd.read_csv(
-            resourcefilepath / 'healthsystem' / 'consumables' / 'ResourceFile_Consumables_availability_small.csv'),
+        pd.read_csv(path_to_file / 'ResourceFile_Consumables_availability_small.csv'),
+        fac_ids=fac_ids
+    )
+    check_format_of_consumables_file(
+        pd.read_csv(path_to_file / 'ResourceFile_Consumables_availability_small_original.csv'),
         fac_ids=fac_ids
     )