Skip to content

Commit 9fb6875

Browse files
damian-molinskidamian-molinski
committed
Merge branch 'feat/database_optimization_3528' into feat/campaign_total_ask
2 parents d977e5c + 56e62fc commit 9fb6875

File tree

328 files changed

+75056
-2658
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

328 files changed

+75056
-2658
lines changed

‎.config/dictionaries/project.dic‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,7 @@ headful
150150
headlessui
151151
healthz
152152
HIDPI
153+
hotfixes
153154
hotspots
154155
httpx
155156
icudtl
@@ -308,8 +309,8 @@ ripgrep
308309
ristretto
309310
rlib
310311
rngs
311-
rolname
312312
rollouts
313+
rolname
313314
RPATH
314315
ruleset
315316
rustc
@@ -427,4 +428,4 @@ xprv
427428
xpub
428429
xpublic
429430
xvfb
430-
yoroi
431+
yoroi

‎.github/workflows/ci.yml‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,20 +18,27 @@ permissions:
1818

1919
jobs:
2020
paths-filter:
21-
if: ${{ github.event.pull_request }}
2221
runs-on: ubuntu-latest
2322
outputs:
2423
catalyst-gateway: ${{ steps.filter.outputs.catalyst-gateway }}
2524
catalyst-voices: ${{ steps.filter.outputs.catalyst-voices }}
25+
docs: ${{ steps.filter.outputs.docs }}
26+
not-only-docs: ${{ steps.filter.outputs.not-only-docs }}
2627
steps:
27-
- uses: dorny/paths-filter@v2
28+
- uses: actions/checkout@v4
29+
- uses: dorny/paths-filter@v3
2830
id: filter
2931
with:
3032
filters: |
3133
catalyst-voices:
3234
- 'catalyst_voices/**'
3335
catalyst-gateway:
3436
- 'catalyst-gateway/**'
37+
docs:
38+
- 'docs/**'
39+
- '.config/dictionaries/project.dic'
40+
not-only-docs:
41+
- '!((docs/**)|.config/dictionaries/project.dic)'
3542
3643
reject:
3744
if: ${{ !github.event.pull_request.draft }}
@@ -51,9 +58,17 @@ jobs:
5158
\s*FROM.*?:latest
5259
No images should be built using the 'latest' tag
5360
61+
docs-only:
62+
if: ${{ needs.paths-filter.outputs.docs == 'true' && needs.paths-filter.outputs.not-only-docs != 'true' }}
63+
needs: [reject, paths-filter]
64+
uses: input-output-hk/catalyst-forge/.github/workflows/release.yml@ci/v1.10.0
65+
with:
66+
releases: '[{"project":"./docs","name":"docs"}]'
67+
forge_version: 0.21.0
68+
5469
ci:
55-
if: ${{ !github.event.pull_request.draft }}
56-
needs: [reject]
70+
if: ${{ (!github.event.pull_request.draft && needs.paths-filter.outputs.not-only-docs == 'true') || startsWith(github.ref, 'refs/tags/') }}
71+
needs: [reject, paths-filter]
5772
uses: input-output-hk/catalyst-forge/.github/workflows/ci.yml@ci/v1.10.0
5873
with:
5974
forge_version: 0.21.0

‎.github/workflows/gateway-tests.yml‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -106,18 +106,6 @@ jobs:
106106
run: |
107107
docker compose -f catalyst-gateway/tests/docker-compose.yml up api-tests-runner --exit-code-from api-tests-runner
108108
109-
# Skip for now as it adds ~10mins to CI execution time, will try to incorporate this in nightly instead
110-
- name: Integration health thru proxy tests
111-
if: false
112-
env:
113-
API_TEST_MARKS: "health_with_proxy_endpoint"
114-
EVENT_DB_URL: "haproxy:18080"
115-
INDEX_DB_URL: "haproxy:18090"
116-
run: |
117-
docker stop cat-gateway || true
118-
docker compose -f catalyst-gateway/tests/docker-compose.yml up haproxy --detach
119-
docker compose -f catalyst-gateway/tests/docker-compose.yml up api-tests-runner --exit-code-from api-tests-runner
120-
121109
- name: Collect and upload test reports
122110
uses: actions/upload-artifact@v4
123111
if: success() || failure()

‎.github/workflows/nightly-ci.yml‎

Lines changed: 51 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -69,28 +69,65 @@ jobs:
6969
id: gateway-healthy
7070
if: ${{ steps.gateway-started.conclusion == 'success' }}
7171
run: |
72-
echo "Waiting for container to be healthy..."
73-
for i in {1..500}; do
74-
STATUS=$(docker inspect --format='{{.State.Health.Status}}' cat-gateway)
75-
echo "Health status: $STATUS"
76-
if [ "$STATUS" == "healthy" ]; then
77-
echo "Container is healthy!"
78-
break
79-
fi
80-
if [ "$STATUS" == "unhealthy" ]; then
81-
echo "Container became unhealthy."
82-
exit 1
83-
fi
84-
sleep 5
85-
done
72+
echo "Waiting for container to be healthy..."
73+
for i in {1..500}; do
74+
STATUS=$(docker inspect --format='{{.State.Health.Status}}' cat-gateway)
75+
echo "Health status: $STATUS"
76+
if [ "$STATUS" == "healthy" ]; then
77+
echo "Container is healthy!"
78+
break
79+
fi
80+
if [ "$STATUS" == "unhealthy" ]; then
81+
echo "Container became unhealthy."
82+
exit 1
83+
fi
84+
sleep 5
85+
done
8686
8787
- name: Schemathesis tests
88+
id: schemathesis-tests
8889
if: ${{ steps.gateway-healthy.conclusion == 'success' }}
8990
run: |
9091
export HYPOTHESIS_MAX_EXAMPLES=5000
9192
export MAX_RESPONSE_TIME=25000
9293
docker compose -f catalyst-gateway/tests/docker-compose.yml up schemathesis-runner --exit-code-from schemathesis-runner
9394
95+
- name: Spin up catalyst-gateway with haproxy
96+
id: gateway-with-proxy-started
97+
env:
98+
EVENT_DB_URL: "haproxy:18080"
99+
INDEX_DB_URL: "haproxy:18090"
100+
run: |
101+
docker stop cat-gateway || true
102+
docker compose -f catalyst-gateway/tests/docker-compose.yml up haproxy --detach
103+
docker compose -f catalyst-gateway/tests/docker-compose.yml up cat-gateway --detach
104+
105+
- name: Wait for cat-gateway to become healthy
106+
id: gateway-with-proxy-healthy
107+
if: ${{ steps.gateway-with-proxy-started.conclusion == 'success' }}
108+
run: |
109+
echo "Waiting for container to be healthy..."
110+
for i in {1..500}; do
111+
STATUS=$(docker inspect --format='{{.State.Health.Status}}' cat-gateway)
112+
echo "Health status: $STATUS"
113+
if [ "$STATUS" == "healthy" ]; then
114+
echo "Container is healthy!"
115+
break
116+
fi
117+
if [ "$STATUS" == "unhealthy" ]; then
118+
echo "Container became unhealthy."
119+
exit 1
120+
fi
121+
sleep 5
122+
done
123+
124+
- name: Integration health thru proxy tests
125+
if: ${{ steps.gateway-with-proxy-healthy.conclusion == 'success' }}
126+
env:
127+
API_TEST_MARKS: "health_with_proxy_endpoint"
128+
run: |
129+
docker compose -f catalyst-gateway/tests/docker-compose.yml up api-tests-runner --exit-code-from api-tests-runner
130+
94131
slack:
95132
name: slack notification
96133
needs: tests

‎Earthfile‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
VERSION 0.8
22

3-
IMPORT github.com/input-output-hk/catalyst-ci/earthly/mdlint:v3.5.28 AS mdlint-ci
4-
IMPORT github.com/input-output-hk/catalyst-ci/earthly/cspell:v3.5.28 AS cspell-ci
5-
IMPORT github.com/input-output-hk/catalyst-ci/earthly/postgresql:v3.5.28 AS postgresql-ci
3+
IMPORT github.com/input-output-hk/catalyst-ci/earthly/mdlint:v3.6.2 AS mdlint-ci
4+
IMPORT github.com/input-output-hk/catalyst-ci/earthly/cspell:v3.6.2 AS cspell-ci
5+
IMPORT github.com/input-output-hk/catalyst-ci/earthly/postgresql:v3.6.2 AS postgresql-ci
66

77
ARG --global REGISTRY="harbor.shared-services.projectcatalyst.io/dockerhub/library"
88
FROM ${REGISTRY}/debian:stable-slim

‎catalyst-gateway/Cargo.toml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
[workspace]
2-
resolver = "2"
2+
resolver = "3"
33
members = [
44
"bin",
55
]
66

77
[workspace.package]
8-
edition = "2021"
8+
edition = "2024"
99
version = "0.0.1"
1010
authors = [
1111
"Steven Johnson <[email protected]>"

‎catalyst-gateway/Earthfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
VERSION 0.8
22

3-
IMPORT github.com/input-output-hk/catalyst-ci/earthly/rust:v3.5.28 AS rust-ci
3+
IMPORT github.com/input-output-hk/catalyst-ci/earthly/rust:v3.6.2 AS rust-ci
44
IMPORT ../ AS repo-ci
55
IMPORT github.com/input-output-hk/catalyst-voices/catalyst-gateway:main AS cat-gateway-main
66

‎catalyst-gateway/README.md‎

Lines changed: 57 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,14 @@
33
* [Catalyst Data Gateway](#catalyst-data-gateway)
44
* [Code Organization](#code-organization)
55
* [`./bin`](#bin)
6-
* [`./crates`](#crates)
76
* [`./event-db`](#event-db)
87
* [Build and Run](#build-and-run)
98
* [Docker images](#docker-images)
9+
* [Build](#build)
10+
* [Run](#run)
1011
* [Rust binary](#rust-binary)
12+
* [Build](#build-1)
13+
* [Run](#run-1)
1114

1215
The Catalyst Data Gateway is the backend of the Catalyst Voices hosted stack.
1316

@@ -20,11 +23,6 @@ In future it will also act as a gateway from Centralized catalyst infrastructure
2023

2124
This is the main Catalyst Gateway Application.
2225

23-
### `./crates`
24-
25-
These are fully re-usable generalized `rust` crates that the Catalyst Gateway uses and are developed with it.
26-
They are also able to be used stand-alone in other projects and can be published separately.
27-
2826
### `./event-db`
2927

3028
Defines the Postgres Catalyst Event Database that the Catalyst gateway uses for running Catalyst Events.
@@ -40,30 +38,60 @@ or you can build a docker image and run everything with the `docker-compose`.
4038

4139
To build and run docker images follow these steps:
4240

43-
1. Run `earthly +package` to build a cat-gateway docker image.
44-
2. Run `earthly ./event-db+build` to build an event-db docker image.
45-
3. Run `docker-compose up cat-gateway` to spin up cat-gateway with event-db from already built images.
41+
#### Build
42+
43+
* Build `cat-gateway`:
44+
45+
```sh
46+
earthly ./catalyst-gateway+docker
47+
```
48+
49+
* Build `event-db`:
4650

47-
Note that every time when you are building an image it obsoletes an old image but does not remove it,
48-
so don't forget to clean up dangling images of the event-db and cat-gateway in your docker environment.
51+
```sh
52+
earthly ./catalyst-gateway/event-db+docker
53+
```
54+
55+
#### Run
56+
57+
```sh
58+
docker compose -f ./catalyst-gateway/docker-compose.yml up cat-gateway
59+
```
4960

5061
### Rust binary
5162

52-
To build and run a Rust binary follow these steps:
53-
54-
1. Run `cargo build -p cat-gateway --release`
55-
to compile a release version of the cat-gateway
56-
2. Run `earthly ./event-db+build` to build an event-db docker image
57-
3. If you need to have a `preprod-snapshot` unarchive snapshot data to the `/tmp/preprod/` dir.
58-
You can download `preprod-snapshot` from this
59-
[resource](https://mithril.network/explorer/?aggregator=https%3A%2F%2Faggregator.release-preprod.api.mithril.network%2Faggregator).
60-
4. Run
61-
62-
```sh
63-
./target/release/cat-gateway run \
64-
--address "127.0.0.1:3030" \
65-
--database-url=postgres://catalyst-event-dev:CHANGE_ME@localhost/CatalystEventDev \
66-
--log-level=debug \
67-
--log-format=compact \
68-
--metrics-address "127.0.0.1:3032"
69-
```
63+
To build and run `cat-gateway` natively,
64+
as a first step it will be needed to anyway build and run `event-db` and `index-db` as docker containers
65+
66+
#### Build
67+
68+
* Build `cat-gateway`:
69+
70+
```sh
71+
cd catalyst-gateway
72+
cargo b --release
73+
```
74+
75+
* Build `event-db`:
76+
77+
```sh
78+
earthly ./catalyst-gateway/event-db+docker
79+
```
80+
81+
#### Run
82+
83+
* Run `event-db` and `index-db`:
84+
85+
```sh
86+
docker compose -f ./catalyst-gateway/docker-compose.yml up event-db index-db
87+
```
88+
89+
* Run `cat-gateway`:
90+
91+
```sh
92+
export SIGNED_DOC_SK="0x6455585b5dcc565c8975bc136e215d6d4dd96540620f37783c564da3cb3686dd"
93+
export CHAIN_NETWORK="Preprod"
94+
export INTERNAL_API_KEY="123"
95+
export EVENT_DB_URL="postgres://catalyst-event-dev:CHANGE_ME@localhost:5432/CatalystEventDev"
96+
./catalyst-gateway/target/release/cat-gateway run
97+
```

‎catalyst-gateway/bin/Cargo.toml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,12 @@ repository.workspace = true
1515
workspace = true
1616

1717
[dependencies]
18-
cardano-chain-follower = { version = "0.0.17", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "cardano-chain-follower/v0.0.17" }
19-
rbac-registration = { version = "0.0.13", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "rbac-registration/v0.0.13" }
20-
catalyst-signed-doc = { version = "0.0.9", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "catalyst-signed-doc/v0.0.9" }
18+
cardano-chain-follower = { version = "0.0.18", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "cardano-chain-follower/v0.0.18" }
19+
rbac-registration = { version = "0.0.14", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "rbac-registration/v0.0.14" }
20+
catalyst-signed-doc = { version = "0.0.10", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "catalyst-signed-doc/v0.0.10" }
2121
catalyst-signed-doc-v1 = { package = "catalyst-signed-doc", version = "0.0.4", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "catalyst-signed-doc/v.0.0.4" }
2222
c509-certificate = { version = "0.0.3", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "c509-certificate-v0.0.3" }
23-
catalyst-types = { version = "0.0.9", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "catalyst-types/v0.0.9" }
23+
catalyst-types = { version = "0.0.10", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "catalyst-types/v0.0.10" }
2424

2525
clap = { version = "4.5.20", features = ["derive", "env"] }
2626
tracing = { version = "0.1.40", features = ["log"] }

‎catalyst-gateway/bin/src/build_info.rs‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -48,15 +48,15 @@ pub(crate) fn log_build_info() {
4848
let mut branch = "Unknown".to_string();
4949
let mut tags = "Unknown".to_string();
5050

51-
if let Some(ref vc) = info.version_control {
52-
if let Some(git) = vc.git() {
53-
commit_id.clone_from(&git.commit_short_id);
54-
commit_timestamp = git.commit_timestamp.to_rfc3339();
55-
if let Some(git_branch) = git.branch.clone() {
56-
branch = git_branch;
57-
}
58-
tags = git.tags.join(",");
51+
if let Some(ref vc) = info.version_control
52+
&& let Some(git) = vc.git()
53+
{
54+
commit_id.clone_from(&git.commit_short_id);
55+
commit_timestamp = git.commit_timestamp.to_rfc3339();
56+
if let Some(git_branch) = git.branch.clone() {
57+
branch = git_branch;
5958
}
59+
tags = git.tags.join(",");
6060
}
6161

6262
let ipv4 = utilities::net::get_public_ipv4().to_string();
@@ -65,15 +65,15 @@ pub(crate) fn log_build_info() {
6565
let mut interfaces: String = "Unknown".to_string();
6666

6767
// Get local IP address v4 and v6
68-
if let Ok(network_interfaces) = list_afinet_netifas() {
69-
if !network_interfaces.is_empty() {
70-
interfaces.clear();
71-
for iface in network_interfaces {
72-
if !interfaces.is_empty() {
73-
interfaces.push(',');
74-
}
75-
let _ = write!(interfaces, "{}:{}", iface.0, iface.1);
68+
if let Ok(network_interfaces) = list_afinet_netifas()
69+
&& !network_interfaces.is_empty()
70+
{
71+
interfaces.clear();
72+
for iface in network_interfaces {
73+
if !interfaces.is_empty() {
74+
interfaces.push(',');
7675
}
76+
let _ = write!(interfaces, "{}:{}", iface.0, iface.1);
7777
}
7878
}
7979

0 commit comments

Comments
 (0)