Added deriveRawApiResponseDF fix (#1283)

souravbaner-da · Sourav692 · web-flow · commit 724fd9eda42a · 2024-09-04T21:56:18.000-04:00
Co-authored-by: Sourav Banerjee &lt;sourav692@gmail.com&gt;
diff --git a/src/main/scala/com/databricks/labs/overwatch/pipeline/BronzeTransforms.scala b/src/main/scala/com/databricks/labs/overwatch/pipeline/BronzeTransforms.scala
@@ -652,7 +652,7 @@ trait BronzeTransforms extends SparkSessionWrapper {
     }
      if (Helpers.pathExists(tmpClusterEventsErrorPath)) {
       persistErrors(
-        spark.read.json(tmpClusterEventsErrorPath)
+       deriveRawApiResponseDF(spark.read.json(tmpClusterEventsErrorPath))
           .withColumn("from_ts", toTS(col("from_epoch")))
           .withColumn("until_ts", toTS(col("until_epoch"))),
         database,

Original file line number	Diff line number	Diff line change
`@@ -652,7 +652,7 @@ trait BronzeTransforms extends SparkSessionWrapper {`
`652`	`652`	`}`
`653`	`653`	`if (Helpers.pathExists(tmpClusterEventsErrorPath)) {`
`654`	`654`	`persistErrors(`
`655`		`- spark.read.json(tmpClusterEventsErrorPath)`
	`655`	`+ deriveRawApiResponseDF(spark.read.json(tmpClusterEventsErrorPath))`
`656`	`656`	`.withColumn("from_ts", toTS(col("from_epoch")))`
`657`	`657`	`.withColumn("until_ts", toTS(col("until_epoch"))),`
`658`	`658`	`database,`