describing training job call every 30 seconds

Author: mvsusp

src/sagemaker/session.py

@@ -605,7 +605,7 @@ def logs_for_job(self, job_name, wait=False, poll=5):  # noqa: C901 - suppress c
         # Notes:
         # - The JOB_COMPLETE state forces us to do an extra pause and read any items that got to Cloudwatch after
         #   the job was marked complete.
-
+        seconds_since_last_describe_job_call = 30
         while True:
             if len(stream_names) < instance_count:
                 # Log streams are created whenever a container starts writing to stdout/err, so this list
@@ -645,8 +645,12 @@ def logs_for_job(self, job_name, wait=False, poll=5):  # noqa: C901 - suppress c
 
             if state == LogState.JOB_COMPLETE:
                 state = LogState.COMPLETE
+            elif seconds_since_last_describe_job_call < 30:
+                seconds_since_last_describe_job_call += max(poll, 1)
             else:
                 description = self.sagemaker_client.describe_training_job(TrainingJobName=job_name)
+                seconds_since_last_describe_job_call = 0
+
                 status = description['TrainingJobStatus']
 
                 if status == 'Completed' or status == 'Failed':