File tree Expand file tree Collapse file tree 1 file changed +9
-9
lines changed
terraform/account-wide-infrastructure/modules/glue/src Expand file tree Collapse file tree 1 file changed +9
-9
lines changed Original file line number Diff line number Diff line change @@ -51,30 +51,30 @@ def run(self):
5151 raise e
5252
5353 def get_last_run (self ):
54- allRuns = self .glue .get_job_runs (JobName = self .job_name )
55- lastRuntime = None
56- if allRuns ["JobRuns" ]:
57- for i in allRuns ["JobRuns" ]:
54+ all_runs = self .glue .get_job_runs (JobName = self .job_name )
55+ last_runtime = None
56+ if all_runs ["JobRuns" ]:
57+ for i in all_runs ["JobRuns" ]:
5858 if i ["JobRunState" ] == "SUCCEEDED" :
59- lastRuntime = time .mktime (i ["StartedOn" ].timetuple ())
59+ last_runtime = time .mktime (i ["StartedOn" ].timetuple ())
6060 break
6161 else :
6262 continue
6363
64- return lastRuntime
64+ return last_runtime
6565
6666 def extract (self ):
6767 """Extract JSON data from S3"""
6868 self .logger .info (f"Extracting data from { self .source_path } as JSON" )
69- lastRuntime = self .get_last_run ()
69+ last_runtime = self .get_last_run ()
7070 data = {}
7171 for name , schema in self .schemas .items ():
72- if lastRuntime :
72+ if last_runtime :
7373 data [name ] = (
7474 self .spark .read .option ("recursiveFileLookup" , "true" )
7575 .schema (schema )
7676 .json (self .source_path )
77- ).where ((col ("host" ).contains (name )) & (col ("time" ) > lastRuntime ))
77+ ).where ((col ("host" ).contains (name )) & (col ("time" ) > last_runtime ))
7878 else :
7979 data [name ] = (
8080 self .spark .read .option ("recursiveFileLookup" , "true" )
You can’t perform that action at this time.
0 commit comments