Added more Moodle exchange cleaning.

Author: eriq-augustine

lms/backend/moodle/backend_test.py

@@ -36,6 +36,10 @@ def child_class_setup(cls) -> None:
             'auth_password': DEFAULT_USER,
         })
 
+        cls.params_to_skip += [
+            'logintoken',
+        ]
+
         cls.headers_to_skip += [
             'edq-lms-moodle-user',
         ]

lms/testing/serverrunner.py

@@ -11,12 +11,16 @@
 import lms.model.constants
 import lms.util.net
 
-BACKEND_REQUEST_CLEANING_FUNCS: typing.Dict[str, typing.Callable] = {
+BACKEND_REQUEST_CLEANING_FUNCS: typing.Dict[typing.Union[str, None], typing.Callable] = {
     lms.model.constants.BACKEND_TYPE_BLACKBOARD: lms.util.net.clean_blackboard_response,
     lms.model.constants.BACKEND_TYPE_CANVAS: lms.util.net.clean_canvas_response,
     lms.model.constants.BACKEND_TYPE_MOODLE: lms.util.net.clean_moodle_response,
 }
 
+BACKEND_EXCHANGE_FINALIZING_FUNCS: typing.Dict[typing.Union[str, None], typing.Callable] = {
+    lms.model.constants.BACKEND_TYPE_MOODLE: lms.util.net.finalize_moodle_exchange,
+}
+
 class LMSServerRunner(edq.testing.serverrunner.ServerRunner):
     """ A server runner specifically for LMS servers. """
 
@@ -38,6 +42,12 @@ def __init__(self,
         The original value may be changed in start(), and will be reset in stop().
         """
 
+        self._old_exchanges_finalize_func: typing.Union[str, None] = None
+        """
+        The value of edq.util.net._exchanges_finalize_func when start() is called.
+        The original value may be changed in start(), and will be reset in stop().
+        """
+
         self._old_set_exchanges_clean_func: bool = False
         """
         The value of lms.cli.parser._set_exchanges_clean_func when start() is called.
@@ -57,21 +67,21 @@ def __init__(self,
         """
 
     def start(self) -> None:
-        super().start()
-
-        # Resolve the backend type.
-        self.backend_type = lms.backend.instance.guess_backend_type(self.server, backend_type = self.backend_type)
-        if (self.backend_type is None):
-            raise ValueError(f"Unable to determine backend type for server '{self.server}'.")
-
         # Set configs.
 
         exchange_clean_func = BACKEND_REQUEST_CLEANING_FUNCS.get(self.backend_type, lms.util.net.clean_lms_response)
         exchange_clean_func_name = edq.util.reflection.get_qualified_name(exchange_clean_func)
-
         self._old_exchanges_clean_func = edq.util.net._exchanges_clean_func
         edq.util.net._exchanges_clean_func = exchange_clean_func_name
 
+        self._old_exchanges_finalize_func = edq.util.net._exchanges_finalize_func
+        exchange_finalize_func = BACKEND_EXCHANGE_FINALIZING_FUNCS.get(self.backend_type, None)
+        if (exchange_finalize_func is not None):
+            exchange_finalize_func_name = edq.util.reflection.get_qualified_name(exchange_finalize_func)
+            edq.util.net._exchanges_finalize_func = exchange_finalize_func_name
+        else:
+            edq.util.net._exchanges_finalize_func = None
+
         self._old_set_exchanges_clean_func = lms.cli.parser._set_exchanges_clean_func
         lms.cli.parser._set_exchanges_clean_func = False
 
@@ -88,6 +98,9 @@ def _make_request_callback(exchange: edq.util.net.HTTPExchange) -> None:
         self._old_serverrunner_logging_level = logger.level
         logger.setLevel(logging.WARNING)
 
+        # Start the server.
+        super().start()
+
     def stop(self) -> bool:
         if (self._old_serverrunner_logging_level is not None):
             logger = logging.getLogger('edq.testing.serverrunner')

lms/util/net.py

@@ -6,6 +6,7 @@
 import urllib.parse
 
 import edq.util.json
+import edq.util.net
 import requests
 
 import lms.model.constants
@@ -43,7 +44,26 @@
     'vary',
     'x-blackboard-xsrf',
 }
-""" Keys to remove from Blackboard content. """
+""" Keys to remove from Blackboard headers. """
+
+MOODLE_CLEAN_REMOVE_HEADERS: typing.Set[str] = {
+    'accept-ranges',
+    'content-encoding',
+    'content-language',
+    'content-script-type',
+    'content-style-type',
+    'expires',
+    'keep-alive',
+    'last-modified',
+    'pragma',
+    'vary',
+}
+""" Keys to remove from Moodle headers. """
+
+MOODLE_FINALIZE_REMOVE_PARAMS: typing.Set[str] = {
+    'logintoken',
+}
+""" Keys to remove from Moodle headers. """
 
 def clean_lms_response(response: requests.Response, body: str) -> str:
     """
@@ -146,8 +166,22 @@ def clean_moodle_response(response: requests.Response, body: str) -> str:
 
     body = _clean_base_response(response, body)
 
+    # Work on both request and response headers.
+    for headers in [response.headers, response.request.headers]:
+        for key in list(headers.keys()):
+            if (key.strip().lower() in MOODLE_CLEAN_REMOVE_HEADERS):
+                headers.pop(key, None)
+
     return body
 
+def finalize_moodle_exchange(exchange: edq.util.net.HTTPExchange) -> edq.util.net.HTTPExchange:
+    """ Finalize Moodle exhanges. """
+
+    for param in MOODLE_FINALIZE_REMOVE_PARAMS:
+        exchange.parameters.pop(param, None)
+
+    return exchange
+
 def _clean_base_response(response: requests.Response, body: str,
         keep_headers: typing.Union[typing.List[str], None] = None) -> str:
     """

requirements.txt

@@ -1,3 +1,3 @@
 beautifulsoup4>=4.10.0
-edq-utils>=0.2.5
+edq-utils>=0.2.7
 requests>=2.31.0