feature/WIP adding icerberg tables

Author: vitor-pina

src/bronze/get_full_tables.py

@@ -1,5 +1,6 @@
 from pyspark.sql import SparkSession #type:ignore
 from pyspark.sql import DataFrame #type:ignore
+from pyspark.sql.types import StructType, StructField, StringType, IntegerType, FloatType #type:ignore
 import pyspark.sql.functions as F #type:ignore
 import argparse
 import os

src/bronze/sql/course_overviews_courseoverview_ddl.sql

@@ -0,0 +1,2 @@
+-- TDB --
+-- TABLE DDL FOR THE ICEBERG CATALOG --

src/bronze/utils/utils.py

@@ -36,8 +36,8 @@ def get_iceberg_spark_session(self,cfg: Config) -> SparkSession:
             .config("spark.hadoop.fs.s3.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem") \
             .getOrCreate()
         return spark 
-    def get_delta_spark_session(self,S3_ACCESS_KEY: str,S3_SECRET_KEY: str , S3_ENDPOINT: str) -> SparkSession:
 
+    def get_delta_spark_session(self,S3_ACCESS_KEY: str,S3_SECRET_KEY: str , S3_ENDPOINT: str) -> SparkSession:
         spark = SparkSession.builder \
             .appName("incremental_table_ingestion") \
             .config("spark.jars",