Merge pull request #85 from moj-analytical-services/dev

Dev

Author: isichei

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## v2.2.1
+### Change
+- Updated output from `GlueJob` class function `wait_for_job_completion()` (when verbose is set to True), now states how long Glue has been running the job.
+
 ## v2.2.0
 ### Added
 - Fixed bug where glue_specific would not write to json or be a key in dictionary from TableMeta class `to_dict()` method.

etl_manager/etl.py

@@ -403,11 +403,12 @@ def wait_for_completion(self, verbose=False):
 
             status = self.job_status
             status_code = status["JobRun"]["JobRunState"]
-            status_error = status["JobRun"].get("ErrorMessage", "Unknown")
+            status_error = status["JobRun"].get("ErrorMessage", "n/a")
+            exec_time = status["JobRun"].get("ExecutionTime", "n/a")
 
             if verbose:
-                timestamp = datetime.datetime.now().strftime("%Y-%B-%d %H:%M:%S")
-                print("{}: Code: {} | Error: {}".format(timestamp, status_code, status_error))
+                timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                print(f"{timestamp}: Job State: {status_code} | Execution Time: {exec_time} (s) | Error: {status_error}")
 
             if status_code == "SUCCEEDED":
                 break

setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name='etl_manager',
-    version='2.2.0',
+    version='2.2.1',
     packages=find_packages(exclude=['tests*']),
     license='MIT',
     description='A python package to manage etl processes on AWS',