E2E should include Tornado

Author: michael.yak

.github/workflows/python_package_build.yml

@@ -27,7 +27,7 @@ jobs:
       - run: poetry build -vvv
 
       # Install all dependencies except for psutil and run the tests with coverage - this tests handling missing psutil
-      - run: poetry install --extras flask --extras fastapi --extras aiohttp --extras db --extras redis
+      - run: poetry install --extras flask --extras fastapi --extras aiohttp --extras tornado --extras db --extras redis
       - run: make coverage
 
       # Run pylint+mypy after installing psutil so they don't complain on missing dependencies

examples/tornado/tornado_example_app.py

@@ -3,22 +3,26 @@
 import random
 
 from tornado import ioloop
-from tornado.web import Application, RequestHandler
 from tornado.httpserver import HTTPServer
+from tornado.web import Application, RequestHandler
 
 from pyctuator.pyctuator import Pyctuator
 
 my_logger = logging.getLogger("example")
 
+
 class HomeHandler(RequestHandler):
     def get(self):
         my_logger.debug(f"{datetime.datetime.now()} - {str(random.randint(0, 100))}")
-        print("Printing to STDOUT")
         self.write("Hello World!")
 
-app = Application([
-    (r"/", HomeHandler)
-], debug=False)
+
+app = Application(
+    [
+        (r"/", HomeHandler)
+    ],
+    debug=False
+)
 
 example_app_address = "host.docker.internal"
 example_sba_address = "localhost"

pyctuator/impl/aiohttp_pyctuator.py

@@ -122,10 +122,7 @@ async def intercept_requests_and_responses(request: web.Request, handler: Callab
                 web.get("/pyctuator/info", get_info),
                 web.get("/pyctuator/health", get_health),
                 web.get("/pyctuator/metrics", get_metric_names),
-                web.get(
-                    "/pyctuator/metrics/{metric_name}",
-                    get_metric_measurement
-                ),
+                web.get("/pyctuator/metrics/{metric_name}", get_metric_measurement),
                 web.get("/pyctuator/loggers", get_loggers),
                 web.get("/pyctuator/loggers/{logger_name}", get_logger),
                 web.post("/pyctuator/loggers/{logger_name}", set_logger_level),

pyctuator/impl/tornado_pyctuator.py

@@ -1,110 +1,87 @@
 import dataclasses
 import json
-from datetime import datetime
+from datetime import datetime, timedelta
 from functools import partial
-from typing import Any, Optional
+from http import HTTPStatus
+from typing import Any, Optional, Callable
 
 from tornado.web import Application, RequestHandler
 
+from pyctuator.httptrace import TraceRecord, TraceRequest, TraceResponse
+from pyctuator.impl import SBA_V2_CONTENT_TYPE
 from pyctuator.impl.pyctuator_impl import PyctuatorImpl
 from pyctuator.impl.pyctuator_router import PyctuatorRouter
 
 
 # pylint: disable=abstract-method
 class AbstractPyctuatorHandler(RequestHandler):
     pyctuator_router: Optional[PyctuatorRouter] = None
-    dumps: Optional[partial[str]] = None
+    dumps: Optional[Callable[[Any], str]] = None
 
     def initialize(self) -> None:
-        self.pyctuator_router = self.application.settings.get('pyctuator_router')
-        self.dumps = self.application.settings.get('custom_dumps')
+        self.pyctuator_router = self.application.settings.get("pyctuator_router")
+        self.dumps = self.application.settings.get("custom_dumps")
+        self.set_header("Content-Type", SBA_V2_CONTENT_TYPE)
+
+    def options(self) -> None:
+        assert self.pyctuator_router is not None
+        assert self.dumps is not None
+        self.write("")
 
 
 class PyctuatorHandler(AbstractPyctuatorHandler):
     def get(self) -> None:
         assert self.pyctuator_router is not None
         assert self.dumps is not None
-        resp = self.pyctuator_router.get_endpoints_data()
-        self.write(self.dumps(resp))
+        self.write(self.dumps(self.pyctuator_router.get_endpoints_data()))
 
 
 # GET /env
 class EnvHandler(AbstractPyctuatorHandler):
-    def options(self) -> None:
-        assert self.pyctuator_router is not None
-        assert self.dumps is not None
-        self.write('')
-
     def get(self) -> None:
         assert self.pyctuator_router is not None
         assert self.dumps is not None
-        resp = self.pyctuator_router.pyctuator_impl.get_environment()
-        self.write(self.dumps(resp))
+        self.write(self.dumps(self.pyctuator_router.pyctuator_impl.get_environment()))
 
 
 # GET /info
 class InfoHandler(AbstractPyctuatorHandler):
-    def options(self) -> None:
-        assert self.pyctuator_router is not None
-        assert self.dumps is not None
-        self.write('')
-
     def get(self) -> None:
         assert self.pyctuator_router is not None
         assert self.dumps is not None
-        resp = self.pyctuator_router.pyctuator_impl.app_info
-        self.write(self.dumps(resp))
+        self.write(self.dumps(self.pyctuator_router.pyctuator_impl.app_info))
 
 
 # GET /health
 class HealthHandler(AbstractPyctuatorHandler):
-    def options(self) -> None:
-        assert self.pyctuator_router is not None
-        assert self.dumps is not None
-        self.write('')
-
     def get(self) -> None:
         assert self.pyctuator_router is not None
         assert self.dumps is not None
-        resp = self.pyctuator_router.pyctuator_impl.get_health()
-        self.write(self.dumps(resp))
+        self.write(self.dumps(self.pyctuator_router.pyctuator_impl.get_health()))
 
 
 # GET /metrics
 class MetricsHandler(AbstractPyctuatorHandler):
-    def options(self) -> None:
-        assert self.pyctuator_router is not None
-        assert self.dumps is not None
-        self.write('')
-
     def get(self) -> None:
         assert self.pyctuator_router is not None
         assert self.dumps is not None
-        resp = self.pyctuator_router.pyctuator_impl.get_metric_names()
-        self.write(self.dumps(resp))
+        self.write(self.dumps(self.pyctuator_router.pyctuator_impl.get_metric_names()))
 
 
 # GET "/metrics/{metric_name}"
 class MetricsNameHandler(AbstractPyctuatorHandler):
     def get(self, metric_name: str) -> None:
         assert self.pyctuator_router is not None
         assert self.dumps is not None
-        resp = self.pyctuator_router.pyctuator_impl.get_metric_measurement(metric_name)
-        self.write(self.dumps(resp))
+        self.write(self.dumps(self.pyctuator_router.pyctuator_impl.get_metric_measurement(metric_name)))
 
 
 # GET /loggers
 class LoggersHandler(AbstractPyctuatorHandler):
-    def options(self) -> None:
-        assert self.pyctuator_router is not None
-        assert self.dumps is not None
-        self.write('')
-
     def get(self) -> None:
         assert self.pyctuator_router is not None
         assert self.dumps is not None
-        resp = self.pyctuator_router.pyctuator_impl.logging.get_loggers()
-        self.write(self.dumps(resp))
+        self.write(self.dumps(self.pyctuator_router.pyctuator_impl.logging.get_loggers()))
 
 
 # GET /loggers/{logger_name}
@@ -113,16 +90,50 @@ class LoggersNameHandler(AbstractPyctuatorHandler):
     def get(self, logger_name: str) -> None:
         assert self.pyctuator_router is not None
         assert self.dumps is not None
-        resp = self.pyctuator_router.pyctuator_impl.logging.get_logger(logger_name)
-        self.write(self.dumps(resp))
+        self.write(self.dumps(self.pyctuator_router.pyctuator_impl.logging.get_logger(logger_name)))
 
     def post(self, logger_name: str) -> None:
         assert self.pyctuator_router is not None
         assert self.dumps is not None
-        body_str = self.request.body.decode('utf-8')
+        body_str = self.request.body.decode("utf-8")
         body = json.loads(body_str)
-        self.pyctuator_router.pyctuator_impl.logging.set_logger_level(logger_name, body.get('configuredLevel', None))
-        self.write('')
+        self.pyctuator_router.pyctuator_impl.logging.set_logger_level(logger_name, body.get("configuredLevel", None))
+        self.write("")
+
+
+# GET /threaddump
+class ThreadDumpHandler(AbstractPyctuatorHandler):
+    def get(self) -> None:
+        assert self.pyctuator_router is not None
+        assert self.dumps is not None
+        self.write(self.dumps(self.pyctuator_router.pyctuator_impl.get_thread_dump()))
+
+
+# GET /logfile
+class LogFileHandler(AbstractPyctuatorHandler):
+    def get(self) -> None:
+        assert self.pyctuator_router is not None
+        assert self.dumps is not None
+
+        range_header = self.request.headers.get("range")
+        if not range_header:
+            self.write(f"{self.pyctuator_router.pyctuator_impl.logfile.log_messages.get_range()}")
+
+        else:
+            str_res, start, end = self.pyctuator_router.pyctuator_impl.logfile.get_logfile(range_header)
+            self.set_status(HTTPStatus.PARTIAL_CONTENT.value)
+            self.add_header("Content-Type", "text/html; charset=UTF-8")
+            self.add_header("Accept-Ranges", "bytes")
+            self.add_header("Content-Range", f"bytes {start}-{end}/{end}")
+            self.write(str_res)
+
+
+# GET /httptrace
+class HttpTraceHandler(AbstractPyctuatorHandler):
+    def get(self) -> None:
+        assert self.pyctuator_router is not None
+        assert self.dumps is not None
+        self.write(self.dumps(self.pyctuator_router.pyctuator_impl.http_tracer.get_httptrace()))
 
 
 # pylint: disable=too-many-locals,unused-argument
@@ -136,16 +147,51 @@ def __init__(self, app: Application, pyctuator_impl: PyctuatorImpl) -> None:
 
         app.settings.setdefault("pyctuator_router", self)
         app.settings.setdefault("custom_dumps", custom_dumps)
-        app.add_handlers(".*$", [
-            (r"/pyctuator", PyctuatorHandler),
-            (r"/pyctuator/env", EnvHandler),
-            (r"/pyctuator/info", InfoHandler),
-            (r"/pyctuator/health", HealthHandler),
-            (r"/pyctuator/metrics", MetricsHandler),
-            (r"/pyctuator/metrics/(?P<metric_name>.*$)", MetricsNameHandler),
-            (r"/pyctuator/loggers", LoggersHandler),
-            (r"/pyctuator/loggers/(?P<logger_name>.*$)", LoggersNameHandler),
-        ])
+
+        # Register a log-function that records request and response in traces and than delegates to the original func
+        self.delegate_log_function = app.settings.get("log_function")
+        app.settings.setdefault("log_function", self._intercept_request_and_response)
+
+        app.add_handlers(
+            ".*$",
+            [
+                (r"/pyctuator", PyctuatorHandler),
+                (r"/pyctuator/env", EnvHandler),
+                (r"/pyctuator/info", InfoHandler),
+                (r"/pyctuator/health", HealthHandler),
+                (r"/pyctuator/metrics", MetricsHandler),
+                (r"/pyctuator/metrics/(?P<metric_name>.*$)", MetricsNameHandler),
+                (r"/pyctuator/loggers", LoggersHandler),
+                (r"/pyctuator/loggers/(?P<logger_name>.*$)", LoggersNameHandler),
+                (r"/pyctuator/dump", ThreadDumpHandler),
+                (r"/pyctuator/threaddump", ThreadDumpHandler),
+                (r"/pyctuator/logfile", LogFileHandler),
+                (r"/pyctuator/trace", HttpTraceHandler),
+                (r"/pyctuator/httptrace", HttpTraceHandler),
+            ]
+        )
+
+    def _intercept_request_and_response(self, handler: RequestHandler) -> None:
+        # Record the request and response
+        record = TraceRecord(
+            timestamp=datetime.now() - timedelta(seconds=handler.request.request_time()),
+            principal=None,
+            session=None,
+            request=TraceRequest(
+                method=handler.request.method or "",
+                uri=handler.request.full_url(),
+                headers={k.lower(): v for k, v in handler.request.headers.items()}
+            ),
+            response=TraceResponse(
+                status=handler.get_status(),
+                headers={k.lower(): [v] for k, v in handler._headers.items()}  # pylint: disable=protected-access
+            ),
+            timeTaken=int(handler.request.request_time() * 1000),
+        )
+        self.pyctuator_impl.http_tracer.add_record(record)
+
+        if self.delegate_log_function:
+            self.delegate_log_function(handler)
 
     def _custom_json_serializer(self, value: Any) -> Any:
         if dataclasses.is_dataclass(value):

tests/test_pyctuator_e2e.py

@@ -22,9 +22,12 @@
 
 
 # mypy: ignore_errors
+from tests.tornado_test_server import TornadoPyctuatorServer
+
+
 @pytest.fixture(
-    params=[FastApiPyctuatorServer, FlaskPyctuatorServer, AiohttpPyctuatorServer],
-    ids=["FastAPI", "Flask", "aiohttp"]
+    params=[FastApiPyctuatorServer, FlaskPyctuatorServer, AiohttpPyctuatorServer, TornadoPyctuatorServer],
+    ids=["FastAPI", "Flask", "aiohttp", "Tornado"]
 )
 def pyctuator_server(request) -> Generator:  # type: ignore
     # Start a the web-server in which the pyctuator is integrated