Updated report-error process

- Distinguished error-types during URL-checking
- Refactored some app-method calls
- Added an on-error call to handle unexpected errors during report-URL-processing

Author: jecarr

threadcomponents/service/rest_svc.py

@@ -254,9 +254,11 @@ async def _insert_batch_reports(self, request, batch, row_count, token=None):
                 if "#" in url:
                     url = url[: url.index("#")]
                 await self.web_svc.verify_url(request, url=url)
+
             # Raised if verify_url() fails
-            except ValueError as ve:
-                return dict(error=str(ve), alert_user=1)
+            except (SystemError, ValueError) as ve:
+                error_prefix = "URL checks failed:" if isinstance(ve, ValueError) else "System-error:"
+                return dict(error=f"{error_prefix} {ve}", alert_user=1)
 
             # Ensure the report has a unique title
             title = await self.data_svc.get_unique_title(title)
@@ -372,22 +374,27 @@ async def check_queue(self):
         """
         logging.info("CHECKING QUEUE")
         self.clean_current_tasks()
+
         while self.queue.qsize() > 0:  # while there are still tasks to do...
             logging.info("QUEUE SIZE: " + str(self.queue.qsize()))
             await asyncio.sleep(1)  # allow other tasks to run while waiting
+
             while len(self.current_tasks) >= self.MAX_TASKS:  # check resource pool until a task is finished
                 self.clean_current_tasks()
                 await asyncio.sleep(1)  # allow other tasks to run while waiting
+
             criteria = await self.queue.get()  # get next task off queue and run it
             # Use run_in_executor (due to event loop potentially blocked otherwise) to start analysis
             loop = asyncio.get_running_loop()
+
             try:
                 task = loop.run_in_executor(None, partial(self.run_start_analysis, criteria=criteria))
                 self.current_tasks.append(task)
                 await task
+
             except Exception as e:
                 logging.error("Report analysis failed: " + str(e))
-                await self.error_report(criteria)
+                await self.error_report(criteria, high_severity=True)
                 continue
 
     def run_start_analysis(self, criteria=None):
@@ -401,13 +408,16 @@ def run_start_analysis(self, criteria=None):
         finally:
             loop.close()
 
-    async def error_report(self, report):
+    async def error_report(self, report, high_severity=False):
         """Function to error a given report."""
         report_id = report[UID]
         await self.dao.update("reports", where=dict(uid=report_id), data=dict(error=self.dao.db_true_val))
         self.remove_report_from_queue_map(report)
         await self.remove_report_if_automatically_generated(report_id)
 
+        if high_severity:
+            await self.web_svc.on_report_error(None)
+
     async def start_analysis(self, criteria=None):
         report_id = criteria[UID]
         logging.info("Beginning analysis for " + report_id)

threadcomponents/service/web_svc.py

@@ -113,63 +113,94 @@ def check_and_clear_cached_responses(self):
         if len(self.cached_responses) > 100:
             self.cached_responses = dict()
 
+    async def _call_app_method(
+        self,
+        request,
+        *args,
+        method_name=None,
+        return_val_when_local=None,
+        return_val_on_error=None,
+        **kwargs,
+    ):
+        """Function to call an app-method given its name."""
+        if self.is_local:
+            # Not applicable to local setups
+            return return_val_when_local
+
+        try:
+            # Attempt to use app's method; log if this couldn't be done
+            app_method = getattr(request.app if request else self.app, method_name)
+            return await app_method(*args, **kwargs)
+
+        except Exception as e:
+            logging.error(f"Misconfigured app: {method_name}() error: {e}")
+            return return_val_on_error
+
     async def action_allowed(self, request, action, context=None):
         """Function to check an action is permitted given a request."""
         if self.is_local:
             # A permission-checker is not implemented for local-use
             # and the user is in control of all their data, so allow the action
             return True
+
         try:
             # Attempt to use app's method to check permission; log if this couldn't be done
             return await request.app.permission_checker(request, action, context)
+
         except (TypeError, AttributeError) as e:
-            logging.error("Misconfigured app: permission_checker() error: " + str(e))
+            logging.error(f"Misconfigured app: permission_checker() error: {e}")
             raise web.HTTPInternalServerError()
 
     async def url_allowed(self, request, url):
         """Function to check a URL is allowed to be submitted."""
         if self.is_local:
             # A URL-checker is not implemented for local-use so allow the action
             return True
+
         try:
             # Attempt to use app's method to check URL; log if this couldn't be done
             return await request.app.url_checker(request, url)
+
         except (TypeError, AttributeError) as e:
-            logging.error("Misconfigured app: url_checker() error: " + str(e))
-            # SystemError makes more sense but we are listening for ValueErrors
-            raise ValueError("Apologies, this URL could not be processed at this time, please contact us.")
+            logging.error(f"Misconfigured app: url_checker() error: {e}")
+            raise SystemError("Apologies, this URL could not be processed at this time, please contact us.")
 
     async def on_report_complete(self, request, report_data):
         """Function to complete any post-complete actions for a report."""
-        if self.is_local:
-            # No post-complete actions needed for local-use
-            return
-        try:
-            # Attempt to use app's on-complete method; log if this couldn't be done
-            return await request.app.on_report_complete(request, report_data)
-        except (TypeError, AttributeError) as e:
-            logging.error("Misconfigured app: on_report_complete() error: " + str(e))
+        return await self._call_app_method(
+            request,
+            request,
+            report_data,
+            method_name="on_report_complete",
+        )
+
+    async def on_report_error(self, request):
+        """Function to complete any submission-error actions for a report."""
+        return await self._call_app_method(
+            request,
+            method_name="on_report_submission_error",
+        )
 
     async def get_current_arachne_user(self, request):
         """Function to obtain the current Arachne username and token given a request."""
-        if self.is_local:
-            return None, None
-        try:
-            # Attempt to use app's method to obtain the username & token; log if this couldn't be done
-            return await request.app.get_current_arachne_user(request)
-        except (TypeError, AttributeError) as e:
-            logging.error("Misconfigured app: get_current_arachne_user() error: " + str(e))
-            return None, None
+        return await self._call_app_method(
+            request,
+            request,
+            method_name="get_current_arachne_user",
+            return_val_when_local=(None, None),
+            return_val_on_error=(None, None),
+        )
 
     async def auto_gen_data_is_valid(self, request, request_data) -> bool:
         """Function to confirm data for an automatically-generated report is valid."""
-        if self.is_local:
-            return True
-        try:
-            return await request.app.auto_gen_data_is_valid(request, request_data)
-        except Exception as e:
-            logging.error("Misconfigured app: auto_gen_data_is_valid() error: " + str(e))
-        return False
+        return await self._call_app_method(
+            request,
+            request,
+            request_data,
+            method_name="auto_gen_data_is_valid",
+            return_val_when_local=True,
+            return_val_on_error=False,
+        )
 
     async def map_all_html(self, url_input, sentence_limit=None):
         a = newspaper.Article(url_input, keep_article_html=True)