Skip to content

Commit a83e6b6

Browse files
committed
NRL-1320 align to naming conventions
1 parent d9c0f29 commit a83e6b6

File tree

1 file changed

+9
-9
lines changed
  • terraform/account-wide-infrastructure/modules/glue/src

1 file changed

+9
-9
lines changed

terraform/account-wide-infrastructure/modules/glue/src/pipeline.py

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -51,30 +51,30 @@ def run(self):
5151
raise e
5252

5353
def get_last_run(self):
54-
allRuns = self.glue.get_job_runs(JobName=self.job_name)
55-
lastRuntime = None
56-
if allRuns["JobRuns"]:
57-
for i in allRuns["JobRuns"]:
54+
all_runs = self.glue.get_job_runs(JobName=self.job_name)
55+
last_runtime = None
56+
if all_runs["JobRuns"]:
57+
for i in all_runs["JobRuns"]:
5858
if i["JobRunState"] == "SUCCEEDED":
59-
lastRuntime = time.mktime(i["StartedOn"].timetuple())
59+
last_runtime = time.mktime(i["StartedOn"].timetuple())
6060
break
6161
else:
6262
continue
6363

64-
return lastRuntime
64+
return last_runtime
6565

6666
def extract(self):
6767
"""Extract JSON data from S3"""
6868
self.logger.info(f"Extracting data from {self.source_path} as JSON")
69-
lastRuntime = self.get_last_run()
69+
last_runtime = self.get_last_run()
7070
data = {}
7171
for name, schema in self.schemas.items():
72-
if lastRuntime:
72+
if last_runtime:
7373
data[name] = (
7474
self.spark.read.option("recursiveFileLookup", "true")
7575
.schema(schema)
7676
.json(self.source_path)
77-
).where((col("host").contains(name)) & (col("time") > lastRuntime))
77+
).where((col("host").contains(name)) & (col("time") > last_runtime))
7878
else:
7979
data[name] = (
8080
self.spark.read.option("recursiveFileLookup", "true")

0 commit comments

Comments
 (0)