Merge main into feat/sync-with-gateway-v2-endpoints-3362

Author: damian-molinski

.github/workflows/gateway-tests.yml

@@ -106,18 +106,6 @@ jobs:
         run: |
           docker compose -f catalyst-gateway/tests/docker-compose.yml up api-tests-runner --exit-code-from api-tests-runner
 
-# Skip for now as it adds ~10mins to CI execution time, will try to incorporate this in nightly instead
-      - name: Integration health thru proxy tests
-        if: false
-        env:
-          API_TEST_MARKS: "health_with_proxy_endpoint"
-          EVENT_DB_URL: "haproxy:18080"
-          INDEX_DB_URL: "haproxy:18090"
-        run: |
-          docker stop cat-gateway || true
-          docker compose -f catalyst-gateway/tests/docker-compose.yml up haproxy --detach
-          docker compose -f catalyst-gateway/tests/docker-compose.yml up api-tests-runner --exit-code-from api-tests-runner
-
       - name: Collect and upload test reports
         uses: actions/upload-artifact@v4
         if: success() || failure()

.github/workflows/nightly-ci.yml

@@ -69,28 +69,65 @@ jobs:
         id: gateway-healthy
         if: ${{ steps.gateway-started.conclusion == 'success' }}
         run: |
-              echo "Waiting for container to be healthy..."
-              for i in {1..500}; do
-                STATUS=$(docker inspect --format='{{.State.Health.Status}}' cat-gateway)
-                echo "Health status: $STATUS"
-                if [ "$STATUS" == "healthy" ]; then
-                  echo "Container is healthy!"
-                  break
-                fi
-                if [ "$STATUS" == "unhealthy" ]; then
-                  echo "Container became unhealthy."
-                  exit 1
-                fi
-                sleep 5
-              done
+          echo "Waiting for container to be healthy..."
+          for i in {1..500}; do
+            STATUS=$(docker inspect --format='{{.State.Health.Status}}' cat-gateway)
+            echo "Health status: $STATUS"
+            if [ "$STATUS" == "healthy" ]; then
+              echo "Container is healthy!"
+              break
+            fi
+            if [ "$STATUS" == "unhealthy" ]; then
+              echo "Container became unhealthy."
+              exit 1
+            fi
+            sleep 5
+          done
 
       - name: Schemathesis tests
+        id: schemathesis-tests
         if: ${{ steps.gateway-healthy.conclusion == 'success' }}
         run: |
           export HYPOTHESIS_MAX_EXAMPLES=5000
           export MAX_RESPONSE_TIME=25000
           docker compose -f catalyst-gateway/tests/docker-compose.yml up schemathesis-runner --exit-code-from schemathesis-runner
 
+      - name: Spin up catalyst-gateway with haproxy
+        id: gateway-with-proxy-started
+        env:
+          EVENT_DB_URL: "haproxy:18080"
+          INDEX_DB_URL: "haproxy:18090"
+        run: |
+          docker stop cat-gateway || true
+          docker compose -f catalyst-gateway/tests/docker-compose.yml up haproxy --detach
+          docker compose -f catalyst-gateway/tests/docker-compose.yml up cat-gateway --detach
+
+      - name: Wait for cat-gateway to become healthy
+        id: gateway-with-proxy-healthy
+        if: ${{ steps.gateway-with-proxy-started.conclusion == 'success' }}
+        run: |
+          echo "Waiting for container to be healthy..."
+          for i in {1..500}; do
+            STATUS=$(docker inspect --format='{{.State.Health.Status}}' cat-gateway)
+            echo "Health status: $STATUS"
+            if [ "$STATUS" == "healthy" ]; then
+              echo "Container is healthy!"
+              break
+            fi
+            if [ "$STATUS" == "unhealthy" ]; then
+              echo "Container became unhealthy."
+              exit 1
+            fi
+            sleep 5
+          done
+
+      - name: Integration health thru proxy tests
+        if: ${{ steps.gateway-with-proxy-healthy.conclusion == 'success' }}
+        env:
+          API_TEST_MARKS: "health_with_proxy_endpoint"
+        run: |
+          docker compose -f catalyst-gateway/tests/docker-compose.yml up api-tests-runner --exit-code-from api-tests-runner
+
   slack:
     name: slack notification
     needs: tests

Earthfile

@@ -1,8 +1,8 @@
 VERSION 0.8
 
-IMPORT github.com/input-output-hk/catalyst-ci/earthly/mdlint:v3.6.1 AS mdlint-ci
-IMPORT github.com/input-output-hk/catalyst-ci/earthly/cspell:v3.6.1 AS cspell-ci
-IMPORT github.com/input-output-hk/catalyst-ci/earthly/postgresql:v3.6.1 AS postgresql-ci
+IMPORT github.com/input-output-hk/catalyst-ci/earthly/mdlint:v3.6.3 AS mdlint-ci
+IMPORT github.com/input-output-hk/catalyst-ci/earthly/cspell:v3.6.3 AS cspell-ci
+IMPORT github.com/input-output-hk/catalyst-ci/earthly/postgresql:v3.6.3 AS postgresql-ci
 
 ARG --global REGISTRY="harbor.shared-services.projectcatalyst.io/dockerhub/library"
 FROM ${REGISTRY}/debian:stable-slim

catalyst-gateway/Earthfile

@@ -1,6 +1,6 @@
 VERSION 0.8
 
-IMPORT github.com/input-output-hk/catalyst-ci/earthly/rust:v3.6.1 AS rust-ci
+IMPORT github.com/input-output-hk/catalyst-ci/earthly/rust:v3.6.3 AS rust-ci
 IMPORT ../ AS repo-ci
 IMPORT github.com/input-output-hk/catalyst-voices/catalyst-gateway:main AS cat-gateway-main
 

catalyst-gateway/README.md

@@ -3,11 +3,14 @@
 * [Catalyst Data Gateway](#catalyst-data-gateway)
     * [Code Organization](#code-organization)
         * [`./bin`](#bin)
-        * [`./crates`](#crates)
         * [`./event-db`](#event-db)
     * [Build and Run](#build-and-run)
         * [Docker images](#docker-images)
+            * [Build](#build)
+            * [Run](#run)
         * [Rust binary](#rust-binary)
+            * [Build](#build-1)
+            * [Run](#run-1)
 
 The Catalyst Data Gateway is the backend of the Catalyst Voices hosted stack.
 
@@ -20,11 +23,6 @@ In future it will also act as a gateway from Centralized catalyst infrastructure
 
 This is the main Catalyst Gateway Application.
 
-### `./crates`
-
-These are fully re-usable generalized `rust` crates that the Catalyst Gateway uses and are developed with it.
-They are also able to be used stand-alone in other projects and can be published separately.
-
 ### `./event-db`
 
 Defines the Postgres Catalyst Event Database that the Catalyst gateway uses for running Catalyst Events.
@@ -40,30 +38,60 @@ or you can build a docker image and run everything with the `docker-compose`.
 
 To build and run docker images follow these steps:
 
-1. Run `earthly +package` to build a cat-gateway docker image.
-2. Run `earthly ./event-db+build` to build an event-db docker image.
-3. Run `docker-compose up cat-gateway` to spin up cat-gateway with event-db from already built images.
+#### Build
+
+* Build `cat-gateway`:
+
+```sh
+earthly ./catalyst-gateway+docker
+```
+
+* Build `event-db`:
 
-Note that every time when you are building an image it obsoletes an old image but does not remove it,
-so don't forget to clean up dangling images of the event-db and cat-gateway in your docker environment.
+```sh
+earthly ./catalyst-gateway/event-db+docker
+```
+
+#### Run
+
+```sh
+docker compose -f ./catalyst-gateway/docker-compose.yml up cat-gateway
+```
 
 ### Rust binary
 
-To build and run a Rust binary follow these steps:
-
-1. Run `cargo build -p cat-gateway --release`
-  to compile a release version of the cat-gateway
-2. Run `earthly ./event-db+build` to build an event-db docker image
-3. If you need to have a `preprod-snapshot` unarchive snapshot data to the `/tmp/preprod/` dir.
-  You can download `preprod-snapshot` from this
-  [resource](https://mithril.network/explorer/?aggregator=https%3A%2F%2Faggregator.release-preprod.api.mithril.network%2Faggregator).
-4. Run
-
-    ```sh
-      ./target/release/cat-gateway run \
-      --address "127.0.0.1:3030" \
-      --database-url=postgres://catalyst-event-dev:CHANGE_ME@localhost/CatalystEventDev \
-      --log-level=debug \
-      --log-format=compact \
-      --metrics-address "127.0.0.1:3032"
-    ```
+To build and run `cat-gateway` natively,
+as a first step it will be needed to anyway build and run `event-db` and `index-db` as docker containers
+
+#### Build
+
+* Build `cat-gateway`:
+
+```sh
+cd catalyst-gateway
+cargo b --release
+```
+
+* Build `event-db`:
+
+```sh
+earthly ./catalyst-gateway/event-db+docker
+```
+
+#### Run
+
+* Run `event-db` and `index-db`:
+
+```sh
+docker compose -f ./catalyst-gateway/docker-compose.yml up event-db index-db
+```
+
+* Run `cat-gateway`:
+
+```sh
+export SIGNED_DOC_SK="0x6455585b5dcc565c8975bc136e215d6d4dd96540620f37783c564da3cb3686dd"
+export CHAIN_NETWORK="Preprod"
+export INTERNAL_API_KEY="123"
+export EVENT_DB_URL="postgres://catalyst-event-dev:CHANGE_ME@localhost:5432/CatalystEventDev"
+./catalyst-gateway/target/release/cat-gateway run
+```

catalyst-gateway/bin/src/service/api/documents/mod.rs

@@ -90,7 +90,7 @@ impl DocumentApi {
         }
     }
 
-    /// Post A Signed Document Index Query for Newer Versions of v0.04.
+    /// Post A Signed Document Index Query (v2).
     ///
     /// Produces a summary of signed documents that meet the criteria
     /// defined in the request body for new signed document versions of v0.04.

catalyst-gateway/bin/src/service/api/documents/post_document_index_query/v2/response.rs

@@ -12,11 +12,10 @@ use crate::{
         self,
         types::{
             array_types::impl_array_types,
+            cardano::collaborators_list::CollaboratorsList,
             document::{
-                doc_ref_v2::{DocumentReferenceListV2, DocumentReferenceV2},
-                doc_type::DocumentType,
-                id::DocumentId,
-                ver::DocumentVer,
+                doc_chain::DocumentChainDocumented, doc_ref_v2::DocumentReferenceListV2,
+                doc_type::DocumentType, id::DocumentId, ver::DocumentVer,
             },
         },
     },
@@ -155,6 +154,8 @@ pub(crate) struct IndexedDocumentVersionV2 {
     /// Document Type that matches the filter
     #[oai(rename = "type")]
     pub doc_type: DocumentType,
+    /// A unique document identifier that matches the filter.
+    pub id: DocumentId,
     /// Document Reference that matches the filter
     #[oai(rename = "ref", skip_serializing_if_is_none)]
     pub doc_ref: Option<DocumentReferenceListV2>,
@@ -167,43 +168,31 @@ pub(crate) struct IndexedDocumentVersionV2 {
     /// Document Parameter Reference that matches the filter
     #[oai(rename = "doc_parameters", skip_serializing_if_is_none)]
     pub parameters: Option<DocumentReferenceListV2>,
+    /// A list of collaborators who can participate in drafting and submitting a document
+    /// that matches the filter.
+    #[oai(skip_serializing_if_is_none)]
+    pub collaborators: Option<CollaboratorsList>,
+    /// A link to a previous document in a chained sequence that matches the filter.
+    #[oai(skip_serializing_if_is_none)]
+    pub chain: Option<DocumentChainDocumented>,
 }
 
 impl Example for IndexedDocumentVersionV2 {
     fn example() -> Self {
         Self {
             ver: Example::example(),
             doc_type: Example::example(),
+            id: Example::example(),
             doc_ref: Some(Example::example()),
             reply: None,
             template: None,
             parameters: None,
+            collaborators: Some(Example::example()),
+            chain: Some(Example::example()),
         }
     }
 }
 
-/// Document Reference for filtered Documents.
-#[derive(NewType, Debug, Clone, From, Into)]
-#[oai(
-    from_multipart = false,
-    from_parameter = false,
-    to_header = false,
-    example = true
-)]
-pub(crate) struct FilteredDocumentReferenceV2(DocumentReferenceV2);
-
-impl From<catalyst_signed_doc::DocumentRef> for FilteredDocumentReferenceV2 {
-    fn from(value: catalyst_signed_doc::DocumentRef) -> Self {
-        Self(value.into())
-    }
-}
-
-impl Example for FilteredDocumentReferenceV2 {
-    fn example() -> Self {
-        Self(Example::example())
-    }
-}
-
 // List of Indexed Document Version Documented
 impl_array_types!(
     IndexedDocumentVersionDocumentedListV2,
@@ -256,6 +245,8 @@ impl TryFrom<SignedDocBody> for IndexedDocumentVersionDocumentedV2 {
         let mut reply = None;
         let mut template = None;
         let mut parameters = None;
+        let mut collaborators = None;
+        let mut chain = None;
         if let Some(json_meta) = doc.metadata() {
             let meta = catalyst_signed_doc::Metadata::from_json(json_meta.clone())?;
 
@@ -264,17 +255,31 @@ impl TryFrom<SignedDocBody> for IndexedDocumentVersionDocumentedV2 {
             reply = meta.reply().cloned().map(Into::into);
             template = meta.template().cloned().map(Into::into);
             parameters = meta.parameters().cloned().map(Into::into);
+            if !meta.collaborators().is_empty() {
+                collaborators = Some(
+                    meta.collaborators()
+                        .iter()
+                        .cloned()
+                        .map(Into::into)
+                        .collect::<Vec<_>>()
+                        .into(),
+                );
+            }
+            chain = meta.chain().map(Into::into);
         }
 
         if let Some(doc_type) = doc_type {
             Ok(IndexedDocumentVersionDocumentedV2(
                 IndexedDocumentVersionV2 {
                     ver: DocumentVer::new_unchecked(doc.ver().to_string()),
                     doc_type: DocumentType::new_unchecked(doc_type.to_string()),
+                    id: DocumentId::new_unchecked(doc.id().to_string()),
                     doc_ref,
                     reply,
                     template,
                     parameters,
+                    collaborators,
+                    chain,
                 },
             ))
         } else {

catalyst-gateway/bin/src/service/common/types/cardano/collaborators_list.rs

@@ -0,0 +1,27 @@
+//! Implement newtype of `RegistrationList`
+
+use poem_openapi::{
+    registry::MetaSchema,
+    types::{Example, ToJSON},
+};
+
+use crate::service::common::types::{
+    array_types::impl_array_types, cardano::catalyst_id::CatalystId,
+};
+
+impl_array_types!(
+    CollaboratorsList,
+    CatalystId,
+    Some(MetaSchema {
+        example: Self::example().to_json(),
+        max_items: Some(100),
+        items: Some(Box::new(CatalystId::schema_ref())),
+        ..MetaSchema::ANY
+    })
+);
+
+impl Example for CollaboratorsList {
+    fn example() -> Self {
+        Self(vec![Example::example()])
+    }
+}

catalyst-gateway/bin/src/service/common/types/cardano/mod.rs

@@ -6,6 +6,7 @@ pub(crate) mod asset_value;
 pub(crate) mod catalyst_id;
 pub(crate) mod cip19_shelley_address;
 pub(crate) mod cip19_stake_address;
+pub(crate) mod collaborators_list;
 pub(crate) mod hash28;
 pub(crate) mod nonce;
 pub(crate) mod query;