File tree Expand file tree Collapse file tree 2 files changed +6
-2
lines changed
terraform/account-wide-infrastructure/modules/glue/src Expand file tree Collapse file tree 2 files changed +6
-2
lines changed Original file line number Diff line number Diff line change 22
33from awsglue .utils import getResolvedOptions
44from pyspark .context import SparkContext
5+ from src .pipeline import LogPipeline
56
67# Get arguments from AWS Glue job
78args = getResolvedOptions (
1415partition_cols = args ["PARTITION_COLS" ].split ("," ) if "PARTITION_COLS" in args else []
1516
1617# Initialize ETL process
17- etl_job = ETLTemplate (
18+ etl_job = LogPipeline (
1819 spark_context = sc ,
1920 source_path = args ["SOURCE_PATH" ],
2021 target_path = args ["TARGET_PATH" ],
Original file line number Diff line number Diff line change 1- class ETLTemplate :
1+ from src .instances import GlueContextSingleton , LoggerSingleton
2+
3+
4+ class LogPipeline :
25 def __init__ (
36 self ,
47 spark_context ,
You can’t perform that action at this time.
0 commit comments