feat(cat-gateway): Add integration test for health/ready endpoint (#2567)

* feat(cat-gateway): add integration test for health/ready endpoint

* fix(cat-gateway): add index db outage test and fix spelling

* fix(cat-gateway): add test wrappers to Earthfile, update poetry.lock

* fix(cat-gateway): remove wrappers code from Earthfile

* cleanup document.post funciton

* fix

---------

Co-authored-by: Mr-Leshiy <[email protected]>

Author: saibatizoku

.config/dictionaries/project.dic

@@ -206,6 +206,7 @@ minicbor
 minijinja
 mithril
 mitigations
+mitmdump
 mocktail
 moderations
 moka
@@ -258,6 +259,7 @@ pubkey
 PUBLICKEY
 pubspec
 pycardano
+pyright
 pytest
 qrcode
 rapidoc

catalyst-gateway/tests/api_tests/Earthfile

@@ -10,8 +10,9 @@ builder:
     COPY --dir ./api .
     COPY --dir ./integration .
     COPY --dir ./scripts .
-    COPY --dir ./utils .
     COPY --dir ./test_data .
+    COPY --dir ./utils .
+    COPY --dir ./wrappers .
     COPY cat-libs-rust+build/mk_signed_doc .
     DO python-ci+BUILDER
 
@@ -22,6 +23,8 @@ test:
     ENV EVENT_DB_TEST_URL "postgres://catalyst-event-dev:CHANGE_ME@localhost/CatalystEventDev"
     ENV CAT_GATEWAY_TEST_URL "http://127.0.0.1:3030"
     ENV CAT_GATEWAY_INTERNAL_API_KEY "123"
+    # for the current setup its not possible to provide a path to the cat-gateway binary, because its running under the container
+    ENV CAT_GATEWAY_EXECUTABLE_PATH "SOME VALUE"
 
     WITH DOCKER \
         --compose docker-compose.yml \

catalyst-gateway/tests/api_tests/api/v1/document.py

@@ -2,6 +2,7 @@
 from api import cat_gateway_endpoint_url
 
 URL = cat_gateway_endpoint_url("api/v1/document")
+INDEX_URL = cat_gateway_endpoint_url("api/v1/document/index")
 
 
 # Signed document GET
@@ -18,6 +19,15 @@ def put(data: str, token: str):
 
 
 # Signed document POST
-def post(document_url: str, filter: dict):
+def post(filter: dict, limit=None, page=None):
     headers = {"Content-Type": "application/json"}
-    return requests.post(f"{URL}{document_url}", headers=headers, json=filter)
+    url = f"{URL}/index"
+    query_params = []
+    if limit is not None:
+        query_params.append(f"limit={limit}")
+    if page is not None:
+        query_params.append(f"page={page}")
+
+    if query_params:
+        url += "?" + "&".join(query_params)
+    return requests.post(url, headers=headers, json=filter)

catalyst-gateway/tests/api_tests/integration/test_health.py

@@ -0,0 +1,79 @@
+import pytest
+import time
+
+from api.v1 import document, rbac
+from typing import Any, Generator
+from utils import health
+from utils.rbac_chain import rbac_chain_factory, Chain
+from wrappers import TestProxy
+from wrappers.cat_gateway import CatGateway
+
+@pytest.fixture
+def event_db_proxy() -> Generator[Any, Any, Any]:
+    proxy = TestProxy("Event DB", 18080, 5432)
+    proxy.start()
+    yield proxy
+    proxy.stop()
+
+
+@pytest.fixture
+def index_db_proxy() -> Generator[Any, Any, Any]:
+    proxy = TestProxy("Index DB", 18090, 9042)
+    proxy.start()
+    yield proxy
+    proxy.stop()
+
+@pytest.fixture
+def cat_gateway_service() -> Generator[Any, Any, Any]:
+    cat = CatGateway()
+    yield cat
+    cat.stop()
+    del cat
+    print("dropped cat-gateway")
+
+
+@pytest.mark.health_endpoint
+def test_ready_endpoint_with_event_db_outage(event_db_proxy, index_db_proxy, cat_gateway_service, rbac_chain_factory):
+    cat_gateway_service.start()
+    time.sleep(2) # wait for cat-gateway API to start
+    health.is_ready() #assertion
+
+    # suspend event db comms
+    event_db_proxy.suspend()
+    # fetch endpoint that uses event db
+    resp = document.post(filter={})
+    assert(resp.status_code == 503), f"Expected document index to fail: {resp.status_code} - {resp.text}"
+    # call ready endpoint, expect 503
+    health.is_not_ready() #assertion
+
+    # re-enable event db
+    event_db_proxy.resume()
+    #Call `ready` endpoint to attempt re-connection that should succeed
+    health.is_ready()
+
+    # fetch endpoint that uses event db
+    resp = document.post(filter={})
+    assert(resp.status_code == 200), f"Expected document index to succeed: {resp.status_code} - {resp.text}"
+
+    # Index DB testing
+    auth_token = rbac_chain_factory(Chain.Role0).auth_token()
+
+    # Not registered stake address lookup
+    # Cardano test data CIP0019
+    # <https://github.com/cardano-foundation/CIPs/blob/master/CIP-0019/README.md>
+    # cspell:disable-next-line
+    stake_address = "stake_test17rphkx6acpnf78fuvxn0mkew3l0fd058hzquvz7w36x4gtcljw6kf"
+    resp = rbac.get(lookup=stake_address, token=auth_token)
+    assert(resp.status_code == 404), f"Expected not registered stake address: {resp.status_code} - {resp.text}"
+
+    # disable index db
+    index_db_proxy.suspend()
+    resp = rbac.get(lookup=stake_address, token=auth_token)
+    assert(resp.status_code == 503), f"Expected RBAC lookup to fail: {resp.status_code} - {resp.text}"
+
+    # call ready endpoint, expect 503
+    health.is_not_ready() #assertion
+
+    #Call `ready` endpoint to attempt re-connection that should succeed
+    index_db_proxy.resume()
+    health.is_ready() #assertion

catalyst-gateway/tests/api_tests/integration/test_signed_doc.py

@@ -220,7 +220,7 @@ def test_proposal_doc(proposal_doc_factory, rbac_chain_factory):
     ), f"Failed to get document: {resp.status_code} - {resp.text}"
 
     # Post a signed document with filter ID
-    resp = document.post("/index", filter={"id": {"eq": proposal_doc_id}})
+    resp = document.post(filter={"id": {"eq": proposal_doc_id}})
     assert (
         resp.status_code == 200
     ), f"Failed to post document: {resp.status_code} - {resp.text}"
@@ -320,7 +320,7 @@ def test_comment_doc(comment_doc_factory, rbac_chain_factory):
     ), f"Failed to get document: {resp.status_code} - {resp.text}"
 
     # Post a signed document with filter ID
-    resp = document.post("/index", filter={"id": {"eq": comment_doc_id}})
+    resp = document.post(filter={"id": {"eq": comment_doc_id}})
     assert (
         resp.status_code == 200
     ), f"Failed to post document: {resp.status_code} - {resp.text}"
@@ -400,7 +400,6 @@ def test_submission_action(submission_action_factory, rbac_chain_factory):
 
     # Post a signed document with filter ID
     resp = document.post(
-        "/index",
         filter={"id": {"eq": submission_action_id}},
     )
     assert (
@@ -543,7 +542,8 @@ def test_document_index_endpoint(
         page = 0
         filter = {"id": {"eq": doc.metadata["id"]}}
         resp = document.post(
-            f"/index?limit={limit}&page={page}",
+            limit=limit,
+            page=page,
             filter=filter,
         )
         assert (
@@ -557,7 +557,8 @@ def test_document_index_endpoint(
 
         page += 1
         resp = document.post(
-            f"/index?limit={limit}&page={page}",
+            limit=limit,
+            page=page,
             filter=filter,
         )
         assert (
@@ -569,7 +570,7 @@ def test_document_index_endpoint(
         assert data["page"]["remaining"] == total_amount - 1 - page
 
         resp = document.post(
-            f"/index?limit={total_amount}",
+            limit=total_amount,
             filter=filter,
         )
         assert (
@@ -582,7 +583,7 @@ def test_document_index_endpoint(
 
         # Pagination out of range
         resp = document.post(
-            "/index?page=92233720368547759",
+            page=92233720368547759,
             filter={},
         )
         assert (