Don't create schema pointing to root folder (#135)

jeppe742 · web-flow · commit 0e13822af1b9 · 2024-01-26T09:40:19.000+01:00
diff --git a/spark_utils/delta_lake/functions.py b/spark_utils/delta_lake/functions.py
@@ -95,7 +95,7 @@ def publish_delta_to_hive(
     """
 
     spark_session.sql(
-        f"CREATE SCHEMA IF NOT EXISTS {publish_schema_name} location 'abfss://{data_path[8:].split('/')[0]}/'"
+        f"CREATE SCHEMA IF NOT EXISTS {publish_schema_name} location 'abfss://{'/'.join(data_path[8:].split('/')[0:-1])}/'"
     )
     if refresh:
         spark_session.sql(f"DROP TABLE IF EXISTS {publish_schema_name}.{publish_table_name}")

Original file line number	Diff line number	Diff line change
`@@ -95,7 +95,7 @@ def publish_delta_to_hive(`
`95`	`95`	`"""`
`96`	`96`
`97`	`97`	`spark_session.sql(`
`98`		`- f"CREATE SCHEMA IF NOT EXISTS {publish_schema_name} location 'abfss://{data_path[8:].split('/')[0]}/'"`
	`98`	`+ f"CREATE SCHEMA IF NOT EXISTS {publish_schema_name} location 'abfss://{'/'.join(data_path[8:].split('/')[0:-1])}/'"`
`99`	`99`	`)`
`100`	`100`	`if refresh:`
`101`	`101`	`spark_session.sql(f"DROP TABLE IF EXISTS {publish_schema_name}.{publish_table_name}")`