diff --git a/.config/dictionaries/project.dic b/.config/dictionaries/project.dic index 0a2d74be760..eaa0779f049 100644 --- a/.config/dictionaries/project.dic +++ b/.config/dictionaries/project.dic @@ -408,6 +408,7 @@ vsync wallclock wasmtime Wconditional +webfactory webos Werror Wireframes diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05a4a9bd840..2ed534988d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,33 +58,35 @@ jobs: \s*FROM.*?:latest No images should be built using the 'latest' tag - docs-only: - if: ${{ needs.paths-filter.outputs.docs == 'true' && needs.paths-filter.outputs.not-only-docs != 'true' }} - needs: [reject, paths-filter] - uses: input-output-hk/catalyst-forge/.github/workflows/release.yml@ci/v1.10.0 - with: - releases: '[{"project":"./docs","name":"docs"}]' - forge_version: 0.21.0 - - ci: - if: ${{ (!github.event.pull_request.draft && needs.paths-filter.outputs.not-only-docs == 'true') || startsWith(github.ref, 'refs/tags/') }} - needs: [reject, paths-filter] - uses: input-output-hk/catalyst-forge/.github/workflows/ci.yml@ci/v1.10.0 - with: - forge_version: 0.21.0 - release_only: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/') }} +# docs-only: +# if: ${{ needs.paths-filter.outputs.docs == 'true' && needs.paths-filter.outputs.not-only-docs != 'true' }} +# needs: [reject, paths-filter] +# uses: input-output-hk/catalyst-forge/.github/workflows/release.yml@ci/v1.10.0 +# with: +# releases: '[{"project":"./docs","name":"docs"}]' +# forge_version: 0.21.0 +# +# ci: +# if: ${{ (!github.event.pull_request.draft && needs.paths-filter.outputs.not-only-docs == 'true') || startsWith(github.ref, 'refs/tags/') }} +# needs: [reject, paths-filter] +# uses: input-output-hk/catalyst-forge/.github/workflows/ci.yml@ci/v1.10.0 +# with: +# forge_version: 0.21.0 +# release_only: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/') }} gateway-integration-tests: - needs: [ci, paths-filter] + needs: [paths-filter] if: ${{ needs.paths-filter.outputs.catalyst-gateway == 'true' || github.base_ref == 'main'}} uses: ./.github/workflows/gateway-tests.yml + secrets: + PRIVATE_SSH_KEY: ${{ secrets.PRIVATE_SSH_KEY }} - voices-integration-tests: - needs: [ci, paths-filter] - if: ${{ needs.paths-filter.outputs.catalyst-voices == 'true' || github.base_ref == 'main' }} - uses: ./.github/workflows/voices-tests.yml + # voices-integration-tests: + # needs: [ci, paths-filter] + # if: ${{ needs.paths-filter.outputs.catalyst-voices == 'true' || github.base_ref == 'main' }} + # uses: ./.github/workflows/voices-tests.yml - test-reporting: - needs: [ci, voices-integration-tests, gateway-integration-tests ] - if: ${{ github.base_ref == 'main' }} - uses: ./.github/workflows/generate-allure-report.yml + # test-reporting: + # needs: [ci, voices-integration-tests, gateway-integration-tests ] + # if: ${{ github.base_ref == 'main' }} + # uses: ./.github/workflows/generate-allure-report.yml diff --git a/.github/workflows/gateway-tests.yml b/.github/workflows/gateway-tests.yml index 890141f0cd6..7c103ecb4dd 100644 --- a/.github/workflows/gateway-tests.yml +++ b/.github/workflows/gateway-tests.yml @@ -2,6 +2,9 @@ name: Catalyst Gateway integration tests on: workflow_call: + secrets: + PRIVATE_SSH_KEY: + required: true concurrency: group: gateway-tests-${{ github.head_ref || github.ref }} @@ -123,3 +126,120 @@ jobs: path: "**/*${{ env.COVERAGE_EXT }}" if-no-files-found: error retention-days: 1 + + devnet-tests: + name: Cat-Gateway devnet integration tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + +# - name: Save SSH key to file +# run: | +# mkdir -p ~/.ssh +# touch ~/.ssh/id_rsa1 +# echo "${{ secrets.PRIVATE_SSH_KEY }}" > ~/.ssh/id_rsa1 +# chmod 600 ~/.ssh/id_rsa1 + +# - name: Free Disk Space (Ubuntu) +# uses: jlumbroso/free-disk-space@main +# with: +# # this might remove tools that are actually needed, +# # if set to "true" but frees about 6 GB +# tool-cache: false +# # all of these default to true, but feel free to set to +# # "false" if necessary for your workflow +# android: true +# dotnet: false +# haskell: false +# large-packages: false +# docker-images: false +# swap-storage: true + +# - name: Setup upterm session +# uses: owenthereal/action-upterm@v1 + + - name: Check if secret is set + run: | + if [ -z "${{ secrets.PRIVATE_SSH_KEY }}" ]; then + echo "Secret is empty or not set" + exit 1 + else + echo "Secret is set" + fi + + - name: Setup SSH Agent + uses: webfactory/ssh-agent@v0.9.0 + with: + ssh-private-key: ${{ secrets.PRIVATE_SSH_KEY }} + + - name: Install Forge + uses: input-output-hk/catalyst-forge/actions/install@ci/v1.10.0 + with: + enable_caching: "true" + version: 0.21.0 + + - name: Setup + uses: input-output-hk/catalyst-forge/actions/setup@ci/v1.10.0 + + - name: Prepare artifacts + id : images + uses: input-output-hk/catalyst-forge/actions/run@ci/v1.10.0 + with: + command: run + args: ./catalyst-gateway/tests/+all-images + + - name: Spin up devnet + if: ${{ steps.images.conclusion == 'success' }} + run: | + docker compose -f catalyst-gateway/tests/docker-compose.yml -f catalyst-gateway/tests/docker-compose-localhost.yml --profile devnet up cat-devnet -d + + - name: Wait for cat-devnet to become healthy + run: | + echo "Waiting for container to be healthy..." + for i in {1..500}; do + STATUS=$(docker inspect --format='{{.State.Health.Status}}' cat-devnet) + echo "Health status: $STATUS" + if [ "$STATUS" == "healthy" ]; then + echo "Container is healthy!" + break + fi + if [ "$STATUS" == "unhealthy" ]; then + echo "Container became unhealthy." + exit 1 + fi + sleep 5 + done + + - name: Spin up catalyst-gateway + id: gateway-started + run: | + docker compose -f catalyst-gateway/tests/docker-compose.yml -f catalyst-gateway/tests/docker-compose-localhost.yml --profile devnet up cat-gateway -d + + - name: Wait for cat-gateway to become healthy + id: gateway-healthy + if: ${{ steps.gateway-started.conclusion == 'success' }} + run: | + echo "Waiting for container to be healthy..." + for i in {1..500}; do + STATUS=$(docker inspect --format='{{.State.Health.Status}}' cat-gateway) + echo "Health status: $STATUS" + if [ "$STATUS" == "healthy" ]; then + echo "Container is healthy!" + break + fi + if [ "$STATUS" == "unhealthy" ]; then + echo "Container became unhealthy." + break + fi + sleep 5 + done + + if [ "$STATUS" != "healthy" ]; then + docker compose -f catalyst-gateway/tests/docker-compose.yml logs cat-gateway + exit 1 + fi + + - name: Schemathesis tests + if: ${{ steps.gateway-healthy.conclusion == 'success' }} + run: | + docker compose -f catalyst-gateway/tests/docker-compose.yml -f catalyst-gateway/tests/docker-compose-localhost.yml --profile devnet up schemathesis-runner --exit-code-from schemathesis-runner diff --git a/.gitignore b/.gitignore index a0ce3bcdfb9..f2a5ecfbfb1 100644 --- a/.gitignore +++ b/.gitignore @@ -133,4 +133,6 @@ browser_extensions/ .env # Playwright -/browser-profiles/ \ No newline at end of file +/browser-profiles/ + +/target \ No newline at end of file diff --git a/catalyst-gateway/tests/Earthfile b/catalyst-gateway/tests/Earthfile index ce4dcd9bcfb..e0b28bef01a 100644 --- a/catalyst-gateway/tests/Earthfile +++ b/catalyst-gateway/tests/Earthfile @@ -3,6 +3,7 @@ VERSION 0.8 IMPORT github.com/input-output-hk/catalyst-ci/earthly/spectral:v3.6.1 AS spectral-ci IMPORT .. AS gateway +IMPORT github.com/input-output-hk/catalyst-qa/catalyst-devnet:devnet/v0.0.2 AS devnet # build all necessary docker images required to run `docker-compose.yml` services all-images: @@ -14,6 +15,7 @@ all-images: BUILD +package-rust-tests-runner BUILD +package-event-db-v1 BUILD +package-haproxy + BUILD devnet+docker # a special package for applying migrations only up to "V2_**" migrations package-event-db-v1: diff --git a/catalyst-gateway/tests/docker-compose-localhost.yml b/catalyst-gateway/tests/docker-compose-localhost.yml new file mode 100644 index 00000000000..c22acb40f48 --- /dev/null +++ b/catalyst-gateway/tests/docker-compose-localhost.yml @@ -0,0 +1,51 @@ +# This docker compose file is a temporary workaround for the devnet environment +# It exposes the services on localhost due to a current limitation in how the +# Mithril aggregator serves snapshots. +# This file can be removed once the aggregator is updated, allowing services +# to communicate over the standard Docker network. + +services: + cat-devnet: + network_mode: host + + event-db: + network_mode: host + profiles: + - devnet + + event-db-v1-migrations: + network_mode: host + profiles: + - devnet + + event-db-migrations: + network_mode: host + profiles: + - devnet + + scylla-node: + command: --smp 2 --memory 1G --overprovisioned 1 --rpc-address 0.0.0.0 --broadcast-rpc-address 127.0.0.1 --api-address 0.0.0.0 --alternator-address 0.0.0.0 + network_mode: host + profiles: + - devnet + + cat-gateway: + environment: + - EVENT_DB_URL=postgres://catalyst-event-dev:CHANGE_ME@localhost/CatalystEventDev + - CASSANDRA_PERSISTENT_URL=localhost:9042 + - CASSANDRA_VOLATILE_URL=localhost:9042 + - CHAIN_NETWORK=Devnet + - CHAIN_FOLLOWER_DEVNET_GENESIS_KEY=5b33322c3235332c3138362c3230312c3137372c31312c3131372c3133352c3138372c3136372c3138312c3138382c32322c35392c3230362c3130352c3233312c3135302c3231352c33302c37382c3231322c37362c31362c3235322c3138302c37322c3133342c3133372c3234372c3136312c36385d + depends_on: + cat-devnet: + condition: service_healthy + network_mode: host + profiles: + - devnet + + schemathesis-runner: + environment: + - API_SPEC=http://localhost:3030/docs/cat-gateway.json + network_mode: host + profiles: + - devnet diff --git a/catalyst-gateway/tests/docker-compose.yml b/catalyst-gateway/tests/docker-compose.yml index 222f4849249..1600ec874c6 100644 --- a/catalyst-gateway/tests/docker-compose.yml +++ b/catalyst-gateway/tests/docker-compose.yml @@ -1,6 +1,21 @@ # cspell: words statusgossip socat services: + + cat-devnet: + image: cat-devnet:latest + container_name: cat-devnet + healthcheck: + #TODO when the issues with block initial sizes is fixed, update the healthcheck https://github.com/input-output-hk/catalyst-voices/issues/3327 + test: ["CMD-SHELL","bash /healthcheck.sh"] + start_period: 60s + interval: 30s + retries: 20 + volumes: + - data:/shared-data + profiles: + - devnet + event-db: image: event-db:latest container_name: event-db @@ -13,11 +28,10 @@ services: - DB_SUPERUSER_PASSWORD=postgres - DB_USER=catalyst-event-dev - DB_USER_PASSWORD=CHANGE_ME - - INIT_AND_DROP_DB=true - WITH_MIGRATIONS=false ports: - - 5432:5432 + - "5432:5432" healthcheck: test: [ @@ -41,7 +55,6 @@ services: - DB_SUPERUSER_PASSWORD=postgres - DB_USER=catalyst-event-dev - DB_USER_PASSWORD=CHANGE_ME - - INIT_AND_DROP_DB=false - WITH_SEED_DATA=. - WITH_MIGRATIONS=true @@ -62,7 +75,6 @@ services: - DB_SUPERUSER_PASSWORD=postgres - DB_USER=catalyst-event-dev - DB_USER_PASSWORD=CHANGE_ME - - INIT_AND_DROP_DB=false - WITH_SEED_DATA=. - WITH_MIGRATIONS=true @@ -75,7 +87,7 @@ services: image: scylladb/scylla:latest container_name: scylla-node restart: unless-stopped - command: --smp 2 --memory 1G --overprovisioned 1 --api-address 0.0.0.0 + command: ${SCYLLA_CMD:--smp 2 --memory 1G --overprovisioned 1 --api-address 0.0.0.0} ports: - 9042:9042 healthcheck: @@ -95,7 +107,7 @@ services: # approximately a 25 minutes of waiting retries: 50 environment: - - EVENT_DB_URL=postgres://catalyst-event-dev:CHANGE_ME@${EVENT_DB_URL:-event-db}/CatalystEventDev + - EVENT_DB_URL=postgres://catalyst-event-dev:CHANGE_ME@${EVENT_DB_HOST:-event-db}/CatalystEventDev - CASSANDRA_PERSISTENT_URL=${INDEX_DB_URL:-scylla-node:9042} - CASSANDRA_VOLATILE_URL=${INDEX_DB_URL:-scylla-node:9042} - CHAIN_NETWORK=${ENV_TARGET:-Preprod} @@ -106,7 +118,7 @@ services: # - LOG_LEVEL=error # - RUST_LOG="error,cat_gateway=debug,cardano_chain_follower=debug,mithril-client=debug" ports: - - 3030:3030 + - "3030:3030" depends_on: event-db-migrations: condition: service_started @@ -173,3 +185,6 @@ services: depends_on: cat-gateway: condition: service_healthy + +volumes: + data: