Merge pull request #348 from The-Strategy-Unit/update_pbm_data_to_use_new_pop_proj_data

Author: tomjemmett

model/data/databricks.py

@@ -226,7 +226,14 @@ def get_op(self) -> pd.DataFrame:
             .withColumn("dataset", F.lit("NATIONAL"))
             .withColumn("sitetret", F.lit("NATIONAL"))
             .groupBy(
-                op.drop("index", "fyear", "attendances", "tele_attendances", "sushrg_trimmed", "imd_quintile").columns
+                op.drop(
+                    "index",
+                    "fyear",
+                    "attendances",
+                    "tele_attendances",
+                    "sushrg_trimmed",
+                    "imd_quintile",
+                ).columns
             )
             .agg(
                 (F.sum("attendances") * self._sample_rate).alias("attendances"),
@@ -269,9 +276,7 @@ def get_birth_factors(self) -> pd.DataFrame:
         """
 
         return (
-            self._spark.read.parquet(
-                "/Volumes/nhp/population_projections/files/birth_data/"
-            )
+            self._spark.read.table("nhp.population_projections.births")
             .filter(F.col("area_code").rlike("^E0[6-9]"))
             .withColumn("sex", F.lit(2))
             .groupBy("projection", "age", "sex")
@@ -289,10 +294,7 @@ def get_demographic_factors(self) -> pd.DataFrame:
         """
 
         return (
-            self._spark.read.parquet(
-                "/Volumes/nhp/population_projections/files/demographic_data/projection=principal_proj"
-            )
-            .withColumn("projection", F.lit("principal_proj"))
+            self._spark.read.table("nhp.population_projections.demographics")
             .filter(F.col("area_code").rlike("^E0[6-9]"))
             .groupBy("projection", "age", "sex")
             .pivot("year")