Merge pull request #37 from SolarEdgeTech/tornado

michaelyaakoby · web-flow · commit cf01e4432ce5 · 2020-10-11T18:38:45.000+03:00
Tornado
diff --git a/.github/workflows/python_package_build.yml b/.github/workflows/python_package_build.yml
@@ -27,7 +27,7 @@ jobs:
       - run: poetry build -vvv
 
       # Install all dependencies except for psutil and run the tests with coverage - this tests handling missing psutil
-      - run: poetry install --extras flask --extras fastapi --extras aiohttp --extras db --extras redis
+      - run: poetry install --extras flask --extras fastapi --extras aiohttp --extras tornado --extras db --extras redis
       - run: make coverage
 
       # Run pylint+mypy after installing psutil so they don't complain on missing dependencies
diff --git a/README.md b/README.md
@@ -7,7 +7,7 @@
 Monitor Python web apps using 
 [Spring Boot Admin](https://github.com/codecentric/spring-boot-admin). 
 
-Pyctuator supports **[Flask](https://palletsprojects.com/p/flask/)**, **[FastAPI](https://fastapi.tiangolo.com/)** and **[aiohttp](docs.aiohttp.org)**. **Django** support is planned as well.
+Pyctuator supports **[Flask](https://palletsprojects.com/p/flask/)**, **[FastAPI](https://fastapi.tiangolo.com/)**, **[aiohttp](docs.aiohttp.org)** and **[Tornado](https://www.tornadoweb.org/)**. **Django** support is planned as well.
 
 The following video shows a FastAPI web app being monitored and controled using Spring Boot Admin.
  
diff --git a/examples/Advanced/pyproject.toml b/examples/Advanced/pyproject.toml
@@ -11,7 +11,7 @@ python = "^3.7"
 psutil = { version = "^5.6" }
 fastapi = { version = "^0.41.0" }
 uvicorn = { version = "^0.9.0" }
-pyctuator = { version = "^0.13" }
+pyctuator = { version = "^0.13.1" }
 sqlalchemy = { version = "^1.3" }
 PyMySQL = { version = "^0.9.3" }
 cryptography = { version = "^2.8" }
diff --git a/examples/FastAPI/pyproject.toml b/examples/FastAPI/pyproject.toml
@@ -11,7 +11,7 @@ python = "^3.7"
 psutil = { version = "^5.6" }
 fastapi = { version = "^0.41.0" }
 uvicorn = { version = "^0.9.0" }
-pyctuator = { version = "^0.13" }
+pyctuator = { version = "^0.13.1" }
 
 [build-system]
 requires = ["poetry>=0.12"]
diff --git a/examples/Flask/pyproject.toml b/examples/Flask/pyproject.toml
@@ -10,7 +10,7 @@ authors = [
 python = "^3.7"
 psutil = { version = "^5.6" }
 flask = { version = "^1.1" }
-pyctuator = { version = "^0.13" }
+pyctuator = { version = "^0.13.1" }
 
 [build-system]
 requires = ["poetry>=0.12"]
diff --git a/examples/aiohttp/pyproject.toml b/examples/aiohttp/pyproject.toml
@@ -10,7 +10,7 @@ authors = [
 python = "^3.7"
 psutil = { version = "^5.6" }
 aiohttp = { version = "^3.5.4" }
-pyctuator = { version = "^0.13" }
+pyctuator = { version = "^0.13.1" }
 
 [build-system]
 requires = ["poetry>=0.12"]
diff --git a/examples/images/tornado.png b/examples/images/tornado.png
diff --git a/examples/tornado/README.md b/examples/tornado/README.md
@@ -0,0 +1,17 @@
+# Tornado example
+This example demonstrates the integration with the [Tornado](https://www.tornadoweb.org/).
+
+## Running the example
+1. Start an instance of SBA (Spring Boot Admin):
+    ```sh
+    docker run --rm -p 8080:8080 michayaak/spring-boot-admin:2.2.3-1
+    ```
+2. Once Spring Boot Admin is running, you can run the examples as follow:
+    ```sh
+    cd examples/tornado
+    poetry install
+    poetry run python -m tornado_example_app
+    ``` 
+
+![tornado Example](../images/tornado.png)
+
diff --git a/examples/tornado/pyproject.toml b/examples/tornado/pyproject.toml
@@ -0,0 +1,18 @@
+[tool.poetry]
+name = "tornado-pyctuator-example"
+version = "1.0.0"
+description = "Example of using Pyctuator"
+authors = [
+    "Desmond Stonie <aneasystone@gmail.com>",
+]
+
+[tool.poetry.dependencies]
+python = "^3.7"
+psutil = { version = "^5.6" }
+tornado = { version = "^6.0.4" }
+pyctuator = { version = "^0.13.1" }
+
+[build-system]
+requires = ["poetry>=0.12"]
+build-backend = "poetry.masonry.api"
+
diff --git a/examples/tornado/tornado_example_app.py b/examples/tornado/tornado_example_app.py
@@ -0,0 +1,41 @@
+import datetime
+import logging
+import random
+
+from tornado import ioloop
+from tornado.httpserver import HTTPServer
+from tornado.web import Application, RequestHandler
+
+from pyctuator.pyctuator import Pyctuator
+
+my_logger = logging.getLogger("example")
+
+
+class HomeHandler(RequestHandler):
+    def get(self):
+        my_logger.debug(f"{datetime.datetime.now()} - {str(random.randint(0, 100))}")
+        self.write("Hello World!")
+
+
+app = Application(
+    [
+        (r"/", HomeHandler)
+    ],
+    debug=False
+)
+
+example_app_address = "host.docker.internal"
+example_sba_address = "localhost"
+
+Pyctuator(
+    app,
+    "Tornado Pyctuator",
+    app_url=f"http://{example_app_address}:5000",
+    pyctuator_endpoint_url=f"http://{example_app_address}:5000/pyctuator",
+    registration_url=f"http://{example_sba_address}:8080/instances",
+    app_description="Demonstrate Spring Boot Admin Integration with Tornado",
+)
+
+http_server = HTTPServer(app, decompress_request=True)
+http_server.listen(5000)
+ioloop.IOLoop.current().start()
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyctuator/impl/aiohttp_pyctuator.py b/pyctuator/impl/aiohttp_pyctuator.py
@@ -122,10 +122,7 @@ async def intercept_requests_and_responses(request: web.Request, handler: Callab
                 web.get("/pyctuator/info", get_info),
                 web.get("/pyctuator/health", get_health),
                 web.get("/pyctuator/metrics", get_metric_names),
-                web.get(
-                    "/pyctuator/metrics/{metric_name}",
-                    get_metric_measurement
-                ),
+                web.get("/pyctuator/metrics/{metric_name}", get_metric_measurement),
                 web.get("/pyctuator/loggers", get_loggers),
                 web.get("/pyctuator/loggers/{logger_name}", get_logger),
                 web.post("/pyctuator/loggers/{logger_name}", set_logger_level),
diff --git a/pyctuator/impl/tornado_pyctuator.py b/pyctuator/impl/tornado_pyctuator.py
@@ -0,0 +1,202 @@
+import dataclasses
+import json
+from datetime import datetime, timedelta
+from functools import partial
+from http import HTTPStatus
+from typing import Any, Optional, Callable
+
+from tornado.web import Application, RequestHandler
+
+from pyctuator.httptrace import TraceRecord, TraceRequest, TraceResponse
+from pyctuator.impl import SBA_V2_CONTENT_TYPE
+from pyctuator.impl.pyctuator_impl import PyctuatorImpl
+from pyctuator.impl.pyctuator_router import PyctuatorRouter
+
+
+# pylint: disable=abstract-method
+class AbstractPyctuatorHandler(RequestHandler):
+    pyctuator_router: Optional[PyctuatorRouter] = None
+    dumps: Optional[Callable[[Any], str]] = None
+
+    def initialize(self) -> None:
+        self.pyctuator_router = self.application.settings.get("pyctuator_router")
+        self.dumps = self.application.settings.get("custom_dumps")
+        self.set_header("Content-Type", SBA_V2_CONTENT_TYPE)
+
+    def options(self) -> None:
+        assert self.pyctuator_router is not None
+        assert self.dumps is not None
+        self.write("")
+
+
+class PyctuatorHandler(AbstractPyctuatorHandler):
+    def get(self) -> None:
+        assert self.pyctuator_router is not None
+        assert self.dumps is not None
+        self.write(self.dumps(self.pyctuator_router.get_endpoints_data()))
+
+
+# GET /env
+class EnvHandler(AbstractPyctuatorHandler):
+    def get(self) -> None:
+        assert self.pyctuator_router is not None
+        assert self.dumps is not None
+        self.write(self.dumps(self.pyctuator_router.pyctuator_impl.get_environment()))
+
+
+# GET /info
+class InfoHandler(AbstractPyctuatorHandler):
+    def get(self) -> None:
+        assert self.pyctuator_router is not None
+        assert self.dumps is not None
+        self.write(self.dumps(self.pyctuator_router.pyctuator_impl.app_info))
+
+
+# GET /health
+class HealthHandler(AbstractPyctuatorHandler):
+    def get(self) -> None:
+        assert self.pyctuator_router is not None
+        assert self.dumps is not None
+        self.write(self.dumps(self.pyctuator_router.pyctuator_impl.get_health()))
+
+
+# GET /metrics
+class MetricsHandler(AbstractPyctuatorHandler):
+    def get(self) -> None:
+        assert self.pyctuator_router is not None
+        assert self.dumps is not None
+        self.write(self.dumps(self.pyctuator_router.pyctuator_impl.get_metric_names()))
+
+
+# GET "/metrics/{metric_name}"
+class MetricsNameHandler(AbstractPyctuatorHandler):
+    def get(self, metric_name: str) -> None:
+        assert self.pyctuator_router is not None
+        assert self.dumps is not None
+        self.write(self.dumps(self.pyctuator_router.pyctuator_impl.get_metric_measurement(metric_name)))
+
+
+# GET /loggers
+class LoggersHandler(AbstractPyctuatorHandler):
+    def get(self) -> None:
+        assert self.pyctuator_router is not None
+        assert self.dumps is not None
+        self.write(self.dumps(self.pyctuator_router.pyctuator_impl.logging.get_loggers()))
+
+
+# GET /loggers/{logger_name}
+# POST /loggers/{logger_name}
+class LoggersNameHandler(AbstractPyctuatorHandler):
+    def get(self, logger_name: str) -> None:
+        assert self.pyctuator_router is not None
+        assert self.dumps is not None
+        self.write(self.dumps(self.pyctuator_router.pyctuator_impl.logging.get_logger(logger_name)))
+
+    def post(self, logger_name: str) -> None:
+        assert self.pyctuator_router is not None
+        assert self.dumps is not None
+        body_str = self.request.body.decode("utf-8")
+        body = json.loads(body_str)
+        self.pyctuator_router.pyctuator_impl.logging.set_logger_level(logger_name, body.get("configuredLevel", None))
+        self.write("")
+
+
+# GET /threaddump
+class ThreadDumpHandler(AbstractPyctuatorHandler):
+    def get(self) -> None:
+        assert self.pyctuator_router is not None
+        assert self.dumps is not None
+        self.write(self.dumps(self.pyctuator_router.pyctuator_impl.get_thread_dump()))
+
+
+# GET /logfile
+class LogFileHandler(AbstractPyctuatorHandler):
+    def get(self) -> None:
+        assert self.pyctuator_router is not None
+        assert self.dumps is not None
+
+        range_header = self.request.headers.get("range")
+        if not range_header:
+            self.write(f"{self.pyctuator_router.pyctuator_impl.logfile.log_messages.get_range()}")
+
+        else:
+            str_res, start, end = self.pyctuator_router.pyctuator_impl.logfile.get_logfile(range_header)
+            self.set_status(HTTPStatus.PARTIAL_CONTENT.value)
+            self.add_header("Content-Type", "text/html; charset=UTF-8")
+            self.add_header("Accept-Ranges", "bytes")
+            self.add_header("Content-Range", f"bytes {start}-{end}/{end}")
+            self.write(str_res)
+
+
+# GET /httptrace
+class HttpTraceHandler(AbstractPyctuatorHandler):
+    def get(self) -> None:
+        assert self.pyctuator_router is not None
+        assert self.dumps is not None
+        self.write(self.dumps(self.pyctuator_router.pyctuator_impl.http_tracer.get_httptrace()))
+
+
+# pylint: disable=too-many-locals,unused-argument
+class TornadoHttpPyctuator(PyctuatorRouter):
+    def __init__(self, app: Application, pyctuator_impl: PyctuatorImpl) -> None:
+        super().__init__(app, pyctuator_impl)
+
+        custom_dumps = partial(
+            json.dumps, default=self._custom_json_serializer
+        )
+
+        app.settings.setdefault("pyctuator_router", self)
+        app.settings.setdefault("custom_dumps", custom_dumps)
+
+        # Register a log-function that records request and response in traces and than delegates to the original func
+        self.delegate_log_function = app.settings.get("log_function")
+        app.settings.setdefault("log_function", self._intercept_request_and_response)
+
+        app.add_handlers(
+            ".*$",
+            [
+                (r"/pyctuator", PyctuatorHandler),
+                (r"/pyctuator/env", EnvHandler),
+                (r"/pyctuator/info", InfoHandler),
+                (r"/pyctuator/health", HealthHandler),
+                (r"/pyctuator/metrics", MetricsHandler),
+                (r"/pyctuator/metrics/(?P<metric_name>.*$)", MetricsNameHandler),
+                (r"/pyctuator/loggers", LoggersHandler),
+                (r"/pyctuator/loggers/(?P<logger_name>.*$)", LoggersNameHandler),
+                (r"/pyctuator/dump", ThreadDumpHandler),
+                (r"/pyctuator/threaddump", ThreadDumpHandler),
+                (r"/pyctuator/logfile", LogFileHandler),
+                (r"/pyctuator/trace", HttpTraceHandler),
+                (r"/pyctuator/httptrace", HttpTraceHandler),
+            ]
+        )
+
+    def _intercept_request_and_response(self, handler: RequestHandler) -> None:
+        # Record the request and response
+        record = TraceRecord(
+            timestamp=datetime.now() - timedelta(seconds=handler.request.request_time()),
+            principal=None,
+            session=None,
+            request=TraceRequest(
+                method=handler.request.method or "",
+                uri=handler.request.full_url(),
+                headers={k.lower(): v for k, v in handler.request.headers.items()}
+            ),
+            response=TraceResponse(
+                status=handler.get_status(),
+                headers={k.lower(): [v] for k, v in handler._headers.items()}  # pylint: disable=protected-access
+            ),
+            timeTaken=int(handler.request.request_time() * 1000),
+        )
+        self.pyctuator_impl.http_tracer.add_record(record)
+
+        if self.delegate_log_function:
+            self.delegate_log_function(handler)
+
+    def _custom_json_serializer(self, value: Any) -> Any:
+        if dataclasses.is_dataclass(value):
+            return dataclasses.asdict(value)
+
+        if isinstance(value, datetime):
+            return str(value)
+        return None
diff --git a/pyctuator/pyctuator.py b/pyctuator/pyctuator.py
@@ -53,6 +53,8 @@ def __init__(
 
         * aiohttp - `app` is an instance of `aiohttp.web.Application`
 
+        * Tornado - `app` is an instance of `tornado.web.Application`
+
         :param app: an instance of a supported web-framework with which the pyctuator endpoints will be registered
         :param app_name: the application's name that will be presented in the "Info" section in boot-admin
         :param app_description: a description that will be presented in the "Info" section in boot-admin
@@ -101,6 +103,7 @@ def __init__(
             "flask": self._integrate_flask,
             "fastapi": self._integrate_fastapi,
             "aiohttp": self._integrate_aiohttp,
+            "tornado": self._integrate_tornado
         }
         for framework_name, framework_integration_function in framework_integrations.items():
             if self._is_framework_installed(framework_name):
@@ -195,3 +198,16 @@ def _integrate_aiohttp(self, app: Any, pyctuator_impl: PyctuatorImpl) -> bool:
             AioHttpPyctuator(app, pyctuator_impl)
             return True
         return False
+
+    def _integrate_tornado(self, app: Any, pyctuator_impl: PyctuatorImpl) -> bool:
+        """
+        This method should only be called if we detected that tornado is installed.
+        It will then check whether the given app is a tornado app, and if so - it will add the Pyctuator
+        endpoints to it.
+        """
+        from tornado.web import Application
+        if isinstance(app, Application):
+            from pyctuator.impl.tornado_pyctuator import TornadoHttpPyctuator
+            TornadoHttpPyctuator(app, pyctuator_impl)
+            return True
+        return False
diff --git a/pyproject.toml b/pyproject.toml
diff --git a/tests/test_pyctuator_e2e.py b/tests/test_pyctuator_e2e.py
diff --git a/tests/tornado_test_server.py b/tests/tornado_test_server.py
