Merge pull request #305 from The-Strategy-Unit/59_sushrg-trimmed-apc

yiwen-h · web-flow · commit 022aa8b3b104 · 2025-03-04T11:31:21.000Z
add sushrg_trimmed to apc data
diff --git a/notebooks/extract_data_from_databricks.py b/notebooks/extract_data_from_databricks.py
@@ -36,6 +36,7 @@
     .withColumn("tretspef_raw", F.col("tretspef"))
     .withColumn("fyear", F.floor(F.col("fyear") / 100))
     .withColumn("sex", F.col("sex").cast("int"))
+    .withColumn("sushrg_trimmed", F.expr("substring(sushrg, 1, 4)"))
 )
 
 # COMMAND ----------

Original file line number	Diff line number	Diff line change
`@@ -36,6 +36,7 @@`
`36`	`36`	`.withColumn("tretspef_raw", F.col("tretspef"))`
`37`	`37`	`.withColumn("fyear", F.floor(F.col("fyear") / 100))`
`38`	`38`	`.withColumn("sex", F.col("sex").cast("int"))`
	`39`	`+ .withColumn("sushrg_trimmed", F.expr("substring(sushrg, 1, 4)"))`
`39`	`40`	`)`
`40`	`41`
`41`	`42`	`# COMMAND ----------`