Merge pull request #317 from The-Strategy-Unit/316_update_volumes_pbm

Updates PBM notebook and data.databricks.DatabricksNational

Author: yiwen-h

model/data/databricks.py

@@ -226,7 +226,7 @@ def get_op(self) -> pd.DataFrame:
             .withColumn("dataset", F.lit("NATIONAL"))
             .withColumn("sitetret", F.lit("NATIONAL"))
             .groupBy(
-                op.drop("index", "fyear", "attendances", "tele_attendances").columns
+                op.drop("index", "fyear", "attendances", "tele_attendances", "sushrg_trimmed", "imd_quintile").columns
             )
             .agg(
                 (F.sum("attendances") * self._sample_rate).alias("attendances"),
@@ -270,7 +270,7 @@ def get_birth_factors(self) -> pd.DataFrame:
 
         return (
             self._spark.read.parquet(
-                "/Volumes/su_data/nhp/population-projections/birth_data"
+                "/Volumes/nhp/population_projections/files/birth_data/"
             )
             .filter(F.col("area_code").rlike("^E0[6-9]"))
             .withColumn("sex", F.lit(2))
@@ -290,8 +290,9 @@ def get_demographic_factors(self) -> pd.DataFrame:
 
         return (
             self._spark.read.parquet(
-                "/Volumes/su_data/nhp/population-projections/demographic_data"
+                "/Volumes/nhp/population_projections/files/demographic_data/projection=principal_proj"
             )
+            .withColumn("projection", F.lit("principal_proj"))
             .filter(F.col("area_code").rlike("^E0[6-9]"))
             .groupBy("projection", "age", "sex")
             .pivot("year")
@@ -307,7 +308,7 @@ def get_hsa_activity_table(self) -> pd.DataFrame:
         :rtype: pd.DataFrame
         """
         return (
-            self._spark.read.table("hsa_activity_tables_NATIONAL")
+            self._spark.read.table("nhp.default.hsa_activity_tables_national")
             .filter(F.col("fyear") == self._year * 100 + (self._year + 1) % 100)
             .groupBy("hsagrp", "sex", "age")
             .agg(F.mean("activity").alias("activity"))

notebooks/national_run.py

@@ -28,7 +28,7 @@
 
 # COMMAND ----------
 
-dbutils.widgets.text("data_path", "/Volumes/su_data/nhp/old_nhp_data", "Data Path")
+dbutils.widgets.text("data_path", "/Volumes/nhp/model_data/files", "Data Path")
 dbutils.widgets.text("data_version", "dev", "Data Version")
 dbutils.widgets.text("params_file", "sample_params.json", "Params File")
 dbutils.widgets.text("sample_rate", "0.01", "Sample Rate")
@@ -66,14 +66,8 @@
 
 # COMMAND ----------
 
-# Check that the version is the same in the params and in the data_version variable above
-
-assert dbutils.widgets.get('data_version').rsplit('.', 1)[0] == params["app_version"]
-
-# COMMAND ----------
-
-spark.catalog.setCurrentCatalog("su_data")
-spark.catalog.setCurrentDatabase("nhp")
+spark.catalog.setCurrentCatalog("nhp")
+spark.catalog.setCurrentDatabase("default")
 
 # COMMAND ----------