pass the logger to the function

Author: Tian-2017

scripts/jobs/planning/tascomi_create_daily_snapshot.py

@@ -62,7 +62,7 @@ def deduplicate_by_id_and_last_updated(df):
     return deduplicated_df
 
 
-def prepare_increments(increment_df):
+def prepare_increments(increment_df, logger):
     # In case there are several days worth of increments: only keep the latest version of a record
     id_partition = Window.partitionBy("id")
     # preparation step: create a temporary column to replace NULL last_updated values with 01/01/2020
@@ -233,7 +233,7 @@ def purge_today_partition(
                     )
                     continue
                 # create first snapshot
-                increment_df = prepare_increments(increment_df)
+                increment_df = prepare_increments(increment_df, logger)
                 snapshot_df = increment_df
 
             # snapshot table in glue catalogue
@@ -274,7 +274,7 @@ def purge_today_partition(
                             )
                     else:
                         # prepare COU
-                        increment_df = prepare_increments(increment_df)
+                        increment_df = prepare_increments(increment_df, logger)
                         increment_df = add_snapshot_date_columns(increment_df)
                         # apply COU
                         logger.info(f"Applying increment {increment_table_name}")

terraform/etl/24-aws-glue-tascomi-data.tf

@@ -264,7 +264,7 @@ module "tascomi_create_daily_snapshot" {
   job_name                       = "${local.short_identifier_prefix}tascomi_create_daily_snapshot_planning"
   glue_version                   = "2.0"
   glue_job_worker_type           = "G.2X"
-  number_of_workers_for_glue_job = 12
+  number_of_workers_for_glue_job = 30
   helper_module_key              = data.aws_s3_object.helpers.key
   pydeequ_zip_key                = data.aws_s3_object.pydeequ.key
   spark_ui_output_storage_id     = module.spark_ui_output_storage_data_source.bucket_id