feat: health check (#104)

Author: luislhl

tests/test_api.py

@@ -134,10 +134,19 @@ class BaseAppTestCase(AioHTTPTestCase):
 
     @unittest_run_loop
     async def test_health_check(self):
-        resp = await self.client.request("GET", "/health-check")
+        health_check_result = MagicMock()
+        health_check_result.get_http_status_code.return_value = 200
+        health_check_result.to_json.return_value = {"status": "pass"}
+
+        async def side_effect():
+            return health_check_result
+
+        self.healthcheck.get_health_check.side_effect = side_effect
+
+        resp = await self.client.request("GET", "/health")
         assert resp.status == 200
         data = await resp.json()
-        self.assertTrue(data["success"])
+        self.assertEqual(data, {"status": "pass"})
 
     @unittest_run_loop
     async def test_mining_status(self):
@@ -709,7 +718,8 @@ class AppTestCase(BaseAppTestCase):
 
     async def get_application(self):
         self.manager = TxMiningManager(backend=None, pubsub=MagicMock(), address=None)
-        self.myapp = App(self.manager)
+        self.healthcheck = MagicMock()
+        self.myapp = App(self.manager, self.healthcheck)
         self.version_check = False
         return self.myapp.app
 
@@ -719,8 +729,10 @@ class AppVersionCheckTestCase(BaseAppTestCase):
 
     async def get_application(self):
         self.manager = TxMiningManager(backend=None, pubsub=MagicMock(), address=None)
+        self.healthcheck = MagicMock()
         self.myapp = App(
             self.manager,
+            self.healthcheck,
             min_wallet_desktop_version="0.23.0",
             min_wallet_mobile_version="1.18.3",
             min_wallet_headless_version="0.14.88",

tests/test_healthcheck.py

@@ -0,0 +1,244 @@
+from datetime import datetime
+from unittest.mock import MagicMock, patch
+
+import asynctest  # type: ignore[import]
+
+from txstratum.healthcheck.healthcheck import (
+    ComponentType,
+    FullnodeHealthCheck,
+    HealthCheck,
+    HealthCheckStatus,
+    MiningHealthCheck,
+)
+
+
+class TestFullnodeHealthCheck(asynctest.TestCase):  # type: ignore[misc]
+    def setUp(self) -> None:
+        self.mock_hathor_client = MagicMock()
+        self.fullnode_health_check = FullnodeHealthCheck(
+            backend=self.mock_hathor_client
+        )
+
+    async def test_get_health_check_with_a_healthy_fullnode(self):
+        """Test the response we should generated for a healthy fullnode"""
+        # Mock the implementation of the hathor_client.version.
+        async def side_effect():
+            return {"version": "1.0.0"}
+
+        self.mock_hathor_client.version.side_effect = side_effect
+        self.mock_hathor_client._base_url = "http://localhost:8080"
+
+        result = await self.fullnode_health_check.get_health_check()
+        self.assertEqual(result.component_name, "fullnode")
+        self.assertEqual(result.component_type, ComponentType.FULLNODE)
+        self.assertEqual(result.status, HealthCheckStatus.PASS)
+        self.assertEqual(result.output, "Fullnode is responding correctly")
+        self.assertEqual(result.component_id, "http://localhost:8080")
+
+    async def test_get_health_check_with_an_unhealthy_fullnode(self):
+        """Test the response we should generated for an unhealthy fullnode"""
+        self.mock_hathor_client.version.side_effect = Exception("error")
+        self.mock_hathor_client._base_url = "http://localhost:8080"
+
+        result = await self.fullnode_health_check.get_health_check()
+        self.assertEqual(result.component_name, "fullnode")
+        self.assertEqual(result.component_type, ComponentType.FULLNODE)
+        self.assertEqual(result.status, HealthCheckStatus.FAIL)
+        self.assertEqual(result.output, "Couldn't connect to fullnode: error")
+        self.assertEqual(result.component_id, "http://localhost:8080")
+
+
+class TestMiningHealthCheck(asynctest.TestCase):  # type: ignore[misc]
+    def setUp(self):
+        self.manager = MagicMock()
+        self.mining_health_check = MiningHealthCheck(manager=self.manager)
+
+    async def test_get_health_check_no_miners(self):
+        # Preparation
+        self.manager.has_any_miner.return_value = False
+        # Execution
+        result = await self.mining_health_check.get_health_check()
+        # Assertion
+        self.assertEqual(result.component_name, "manager")
+        self.assertEqual(result.component_type, ComponentType.INTERNAL)
+        self.assertEqual(result.status, HealthCheckStatus.FAIL)
+        self.assertEqual(result.output, "No miners connected")
+
+    async def test_get_health_check_no_submitted_job_in_period(self):
+        # Preparation
+        self.manager.has_any_miner.return_value = True
+        self.manager.has_any_submitted_job_in_period.return_value = False
+        # Execution
+        result = await self.mining_health_check.get_health_check()
+        # Assertion
+        self.assertEqual(result.component_name, "manager")
+        self.assertEqual(result.component_type, ComponentType.INTERNAL)
+        self.assertEqual(result.status, HealthCheckStatus.FAIL)
+        self.assertEqual(result.output, "No miners submitted a job in the last 1 hour")
+
+    async def test_get_health_check_failed_job(self):
+        # Preparation
+        self.manager.has_any_miner.return_value = True
+        self.manager.has_any_submitted_job_in_period.return_value = True
+        job = MagicMock()
+        job.is_failed.return_value = True
+        job.total_time = 9
+        self.manager.tx_jobs = {"job_id": job}
+        # Execution
+        result = await self.mining_health_check.get_health_check()
+        # Assertion
+        self.assertEqual(result.component_name, "manager")
+        self.assertEqual(result.component_type, ComponentType.INTERNAL)
+        self.assertEqual(result.status, HealthCheckStatus.FAIL)
+        self.assertEqual(
+            result.output,
+            "We had 1 failed jobs and 0 long running jobs in the last 5 minutes",
+        )
+
+    async def test_get_health_check_slow_job(self):
+        # Preparation
+        self.manager.has_any_miner.return_value = True
+        self.manager.has_any_submitted_job_in_period.return_value = True
+        job = MagicMock()
+        job.is_failed.return_value = False
+        job.total_time = 11
+        self.manager.tx_jobs = {"job_id": job}
+        # Execution
+        result = await self.mining_health_check.get_health_check()
+        # Assertion
+        self.assertEqual(result.component_name, "manager")
+        self.assertEqual(result.component_type, ComponentType.INTERNAL)
+        self.assertEqual(result.status, HealthCheckStatus.WARN)
+        self.assertEqual(
+            result.output,
+            "We had 0 failed jobs and 1 long running jobs in the last 5 minutes",
+        )
+
+    async def test_get_health_check_ok(self):
+        # Preparation
+        self.manager.has_any_miner.return_value = True
+        self.manager.has_any_submitted_job_in_period.return_value = True
+        job = MagicMock()
+        job.is_failed.return_value = False
+        job.total_time = 9
+        self.manager.tx_jobs = {"job_id": job}
+        # Execution
+        result = await self.mining_health_check.get_health_check()
+        # Assertion
+        self.assertEqual(result.component_name, "manager")
+        self.assertEqual(result.component_type, ComponentType.INTERNAL)
+        self.assertEqual(result.status, HealthCheckStatus.PASS)
+        self.assertEqual(result.output, "Everything is ok")
+
+    # Patch datetime.utcnow() to return a fixed value
+    @patch("txstratum.healthcheck.models.datetime")
+    async def test_return_last_status(self, datetime_mock):
+        """
+        This tests the case where we have no tx_jobs in the last 5 minutes, but we had a previous status of failure.
+
+        We should return the previous status and include its output in the new output.
+        """
+        # Preparation
+        self.manager.has_any_miner.return_value = True
+        self.manager.has_any_submitted_job_in_period.return_value = True
+        self.manager.tx_jobs = {}
+        mock_date = datetime(2021, 1, 1, 0, 0, 0)
+        datetime_mock.utcnow.return_value = mock_date
+        self.mining_health_check.last_manager_status.update(
+            status=HealthCheckStatus.FAIL,
+            output="We had 1 failed jobs and 0 long running jobs in the last 5 minutes",
+        )
+        # Execution
+        result = await self.mining_health_check.get_health_check()
+        # Assertion
+        self.assertEqual(result.component_name, "manager")
+        self.assertEqual(result.component_type, ComponentType.INTERNAL)
+        self.assertEqual(result.status, HealthCheckStatus.FAIL)
+        self.assertEqual(
+            result.output,
+            (
+                "We had no tx_jobs in the last 5 minutes, so we are just returning the last observed status from"
+                f" {mock_date.strftime('%Y-%m-%dT%H:%M:%SZ')}. The output was: We had 1 failed jobs and 0 long"
+                " running jobs in the last 5 minutes"
+            ),
+        )
+
+
+class TestHealthCheck(asynctest.TestCase):  # type: ignore[misc]
+    def setUp(self):
+        self.mock_hathor_client = MagicMock()
+        self.mock_manager = MagicMock()
+
+        self.health_check = HealthCheck(
+            manager=self.mock_manager, backend=self.mock_hathor_client
+        )
+
+    async def test_get_health_check_success(self):
+        """Tests the response we should generate when everything is ok"""
+        # Mock the implementation of the hathor_client.version.
+        async def side_effect():
+            return {"version": "1.0.0"}
+
+        self.mock_hathor_client.version.side_effect = side_effect
+        self.mock_hathor_client._base_url = "http://localhost:8080"
+
+        self.mock_manager.has_any_miner.return_value = True
+        self.mock_manager.has_any_submitted_job_in_period.return_value = True
+        job = MagicMock()
+        job.is_failed.return_value = False
+        job.total_time = 9
+        self.mock_manager.tx_jobs = {"job_id": job}
+
+        result = await self.health_check.get_health_check()
+        self.assertEqual(result.checks["manager"][0].status, HealthCheckStatus.PASS)
+        self.assertEqual(result.checks["manager"][0].output, "Everything is ok")
+        self.assertEqual(result.checks["fullnode"][0].status, HealthCheckStatus.PASS)
+        self.assertEqual(
+            result.checks["fullnode"][0].output, "Fullnode is responding correctly"
+        )
+        self.assertEqual(result.status, HealthCheckStatus.PASS)
+
+    async def test_get_health_check_fullnode_failure(self):
+        """Tests the response we should generate when the fullnode is unhealthy"""
+        self.mock_hathor_client.version.side_effect = Exception("error")
+        self.mock_hathor_client._base_url = "http://localhost:8080"
+
+        self.mock_manager.has_any_miner.return_value = True
+        self.mock_manager.has_any_submitted_job_in_period.return_value = True
+        job = MagicMock()
+        job.is_failed.return_value = False
+        job.total_time = 9
+        self.mock_manager.tx_jobs = {"job_id": job}
+
+        result = await self.health_check.get_health_check()
+        self.assertEqual(result.checks["manager"][0].status, HealthCheckStatus.PASS)
+        self.assertEqual(result.checks["manager"][0].output, "Everything is ok")
+        self.assertEqual(result.checks["fullnode"][0].status, HealthCheckStatus.FAIL)
+        self.assertEqual(
+            result.checks["fullnode"][0].output, "Couldn't connect to fullnode: error"
+        )
+        self.assertEqual(result.status, HealthCheckStatus.FAIL)
+
+    async def test_get_health_check_mining_failure(self):
+        """Tests the response we should generate when the mining is unhealthy"""
+        # Mock the implementation of the hathor_client.version.
+        async def side_effect():
+            return {"version": "1.0.0"}
+
+        self.mock_hathor_client.version.side_effect = side_effect
+        self.mock_hathor_client._base_url = "http://localhost:8080"
+
+        self.mock_manager.has_any_miner.return_value = True
+        self.mock_manager.has_any_submitted_job_in_period.return_value = False
+
+        result = await self.health_check.get_health_check()
+        self.assertEqual(result.checks["manager"][0].status, HealthCheckStatus.FAIL)
+        self.assertEqual(
+            result.checks["manager"][0].output,
+            "No miners submitted a job in the last 1 hour",
+        )
+        self.assertEqual(result.checks["fullnode"][0].status, HealthCheckStatus.PASS)
+        self.assertEqual(
+            result.checks["fullnode"][0].output, "Fullnode is responding correctly"
+        )
+        self.assertEqual(result.status, HealthCheckStatus.FAIL)

txstratum/api.py

@@ -15,6 +15,7 @@
 
 import txstratum.time
 from txstratum.exceptions import JobAlreadyExists, NewJobRefused
+from txstratum.healthcheck.healthcheck import HealthCheck
 from txstratum.jobs import JobStatus, TxJob
 from txstratum.middleware import create_middleware_version_check
 
@@ -45,6 +46,7 @@ class App:
     def __init__(
         self,
         manager: "TxMiningManager",
+        health_check: "HealthCheck",
         *,
         max_tx_weight: Optional[float] = None,
         max_timestamp_delta: Optional[int] = None,
@@ -61,6 +63,7 @@ def __init__(
         super().__init__()
         self.log = logger.new()
         self.manager = manager
+        self.health_check_manager = health_check
         self.max_tx_weight: float = max_tx_weight or MAX_TX_WEIGHT
         self.max_output_script_size = max_output_script_size or MAX_OUTPUT_SCRIPT_SIZE
         self.max_timestamp_delta: float = max_timestamp_delta or MAX_TIMESTAMP_DELTA
@@ -77,13 +80,22 @@ def __init__(
             ]
         )
         self.app.router.add_get("/health-check", self.health_check)
+        self.app.router.add_get("/health", self.health)
         self.app.router.add_get("/mining-status", self.mining_status)
         self.app.router.add_get("/job-status", self.job_status)
         self.app.router.add_post("/submit-job", self.submit_job)
         self.app.router.add_post("/cancel-job", self.cancel_job)
 
         self.fix_invalid_timestamp: bool = fix_invalid_timestamp
 
+    async def health(self, request: web.Request) -> web.Response:
+        """Return the health check status for the tx-mining-service."""
+        health_check_result = await self.health_check_manager.get_health_check()
+        http_status = health_check_result.get_http_status_code()
+
+        return web.json_response(health_check_result.to_json(), status=http_status)
+
+    # XXX: DEPRECATED, Use /health instead
     async def health_check(self, request: web.Request) -> web.Response:
         """Return that the service is running."""
         return web.json_response({"success": True})

txstratum/cli.py

@@ -17,6 +17,7 @@
 
 from txstratum.api import App
 from txstratum.filters import FileFilter, TOIFilter, TXFilter
+from txstratum.healthcheck.healthcheck import HealthCheck
 from txstratum.manager import TxMiningManager
 from txstratum.pubsub import PubSubManager
 from txstratum.toi_client import TOIAsyncClient
@@ -151,6 +152,7 @@ class RunService:
     manager: TxMiningManager
     loop: AbstractEventLoop
     tx_filters: List[TXFilter]
+    health_check: HealthCheck
 
     def __init__(self, args: Namespace) -> None:
         """Initialize the service."""
@@ -167,6 +169,7 @@ def __init__(self, args: Namespace) -> None:
             pubsub=self.pubsub,
             address=args.address,
         )
+        self.health_check = HealthCheck(self.manager, self.backend)
 
     def configure_logging(self, args: Namespace) -> None:
         """Configure logging."""
@@ -253,6 +256,7 @@ def execute(self) -> None:
 
         api_app = App(
             self.manager,
+            self.health_check,
             max_tx_weight=self.args.max_tx_weight,
             max_timestamp_delta=self.args.max_timestamp_delta,
             tx_timeout=self.args.tx_timeout,