Merge branch 'dev' of github.com:dowjones/factiva-analytics-python into dev

Author: miballe

src/factiva/analytics/common/const.py

@@ -96,7 +96,7 @@
     API_JOB_DONE_STATE, API_JOB_FAILED_STATE, API_JOB_CANCELLED_STATE
 ]
 
-API_JOB_ACTIVE_WAIT_SPACING = 10
+API_JOB_ACTIVE_WAIT_SPACING = 15
 
 # SNAPSHOT FILES
 SNAPSHOT_FILE_STATS_FIELDS = [

src/factiva/analytics/integration/files.py

@@ -7,7 +7,7 @@
 class SnapshotFiles(object):
 
 
-    def read_file(self, filepath, stats_only=False, merge_body=False, all_fields=False) -> pd.DataFrame:
+    def read_avro_file(self, filepath, stats_only=False, merge_body=False, all_fields=False) -> pd.DataFrame:
         """Reads a single Dow Jones snapshot datafile
         Parameters
         ----------

src/factiva/analytics/snapshots/extraction.py

@@ -313,7 +313,8 @@ def get_job_response(self) -> bool:
         Returns
         -------
         bool
-            True if the get request was successful. An Exception otherwise.
+            True if the get request was successful. False for FAILED jobs
+            and an Exception for unexpected HTTP codes.
 
         Raises
         ------
@@ -349,6 +350,7 @@ def get_job_response(self) -> bool:
                 self.job_response.errors = response_data['errors']
                 for err in self.job_response.errors:
                     self.__log.error(f"JobError: [{err['title']}] {err['detail']}")
+                return False
         elif response.status_code == 404:
             raise ValueError('Job ID does not exist for the provided user key.')
         elif response.status_code == 400:
@@ -433,7 +435,7 @@ def download_files(self, path=None):
         else:
             print("Job has not yet been submitted")
         self.__log.info('download_files end')
-        return False
+        return False                                                            
 
 
     @log.factiva_logger
@@ -445,13 +447,14 @@ def process_job(self, path=None):  # TODO: Implement Retries if a 500 or timeout
         Returns
         -------
         bool
-            True if the extraction processing was successful. An Exception
-            otherwise.
+            True if the extraction processing was successful. False if the job
+            execution failed. An Exception otherwise.
 
         """
+        ret_val = True
         self.__log.info('process_job Start')
         self.submit_job()
-        self.get_job_response()
+        ret_val = self.get_job_response()
 
         while not (self.job_response.job_state in
                     [const.API_JOB_DONE_STATE,
@@ -460,14 +463,15 @@ def process_job(self, path=None):  # TODO: Implement Retries if a 500 or timeout
             if self.job_response.job_state not in const.API_JOB_EXPECTED_STATES:
                 raise RuntimeError('Unexpected job state')
             time.sleep(const.API_JOB_ACTIVE_WAIT_SPACING)
-            self.get_job_response()
+            if(not self.get_job_response()):
+                ret_val = False
 
         if len(self.job_response.files) > 0:
             self.download_files(path=path)
         else:
             self.__log.info('No files to download. Check for error messages.')
         self.__log.info('process_job End')
-        return True
+        return ret_val
 
 
     def __repr__(self):