use standard logger

Author: Tian-2017

scripts/jobs/planning/tascomi_create_daily_snapshot.py

@@ -1,3 +1,4 @@
+import logging
 import sys
 from datetime import datetime
 
@@ -25,6 +26,11 @@
     table_exists_in_catalog,
 )
 
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger(__name__)
+
 
 def get_latest_snapshot(dfa):
     dfa = dfa.where(col("snapshot_date") == dfa.select(max("snapshot_date")).first()[0])
@@ -62,7 +68,7 @@ def deduplicate_by_id_and_last_updated(df):
     return deduplicated_df
 
 
-def prepare_increments(increment_df, logger):
+def prepare_increments(increment_df):
     # In case there are several days worth of increments: only keep the latest version of a record
     id_partition = Window.partitionBy("id")
     # preparation step: create a temporary column to replace NULL last_updated values with 01/01/2020
@@ -192,7 +198,6 @@ def purge_today_partition(
     sc = SparkContext.getOrCreate()
     glueContext = GlueContext(sc)
     spark = SparkSession(sc)
-    logger = glueContext.get_logger()
     job = Job(glueContext)
     job.init(args["JOB_NAME"], args)
 
@@ -233,7 +238,7 @@ def purge_today_partition(
                     )
                     continue
                 # create first snapshot
-                increment_df = prepare_increments(increment_df, logger)
+                increment_df = prepare_increments(increment_df)
                 snapshot_df = increment_df
 
             # snapshot table in glue catalogue
@@ -274,7 +279,7 @@ def purge_today_partition(
                             )
                     else:
                         # prepare COU
-                        increment_df = prepare_increments(increment_df, logger)
+                        increment_df = prepare_increments(increment_df)
                         increment_df = add_snapshot_date_columns(increment_df)
                         # apply COU
                         logger.info(f"Applying increment {increment_table_name}")