diff --git a/.github/actions/tests/run-e2e-tests/action.yml b/.github/actions/tests/run-e2e-tests/action.yml index 1cd58d8de3..b232ad6e9b 100644 --- a/.github/actions/tests/run-e2e-tests/action.yml +++ b/.github/actions/tests/run-e2e-tests/action.yml @@ -44,6 +44,18 @@ inputs: description: 'Number of threads to use' required: false default: '4' + db_host: + description: 'Override qa_staging_db host' + required: false + db_port: + description: 'Override qa_staging_db port' + required: false + db_sync_host: + description: 'Override db-sync host' + required: false + db_sync_port: + description: 'Override db-sync port' + required: false runs: using: composite @@ -80,6 +92,27 @@ runs: decrypt_switch="--decrypt" fi + # Database override switches + db_host_switch="" + if [ -n "${{ inputs.db_host }}" ]; then + db_host_switch="--db-host '${{ inputs.db_host }}'" + fi + + db_port_switch="" + if [ -n "${{ inputs.db_port }}" ]; then + db_port_switch="--db-port '${{ inputs.db_port }}'" + fi + + db_sync_host_switch="" + if [ -n "${{ inputs.db_sync_host }}" ]; then + db_sync_host_switch="--db-sync-host '${{ inputs.db_sync_host }}'" + fi + + db_sync_port_switch="" + if [ -n "${{ inputs.db_sync_port }}" ]; then + db_sync_port_switch="--db-sync-port '${{ inputs.db_sync_port }}'" + fi + # Define pytest command pytest_cmd="pytest --blockchain '${{ inputs.blockchain }}' \ --env '${{ inputs.env }}' \ @@ -93,6 +126,10 @@ runs: ${latest_mc_epoch_switch} \ ${deployment_mc_epoch_switch} \ ${decrypt_switch} \ + ${db_host_switch} \ + ${db_port_switch} \ + ${db_sync_host_switch} \ + ${db_sync_port_switch} \ --json-report \ --json-report-file=logs/.report.json \ --json-report-summary \ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0155759691..1fec7e168c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -854,12 +854,42 @@ jobs: kubeconfig_base64: ${{ secrets.kubeconfig_base64 }} K8S_SERVER: ${{ secrets.K8S_SERVER }} K8S_SA_TOKEN: ${{ secrets.K8S_SA_TOKEN }} + - name: Install and configure kubectl + run: | + # Install kubectl + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x ./kubectl + sudo mv ./kubectl /usr/local/bin/kubectl + + # Configure kubectl + kubectl config set-cluster my-cluster --server=${{ secrets.K8S_SERVER }} --insecure-skip-tls-verify=true + kubectl config set-credentials github-actions --token=${{ secrets.K8S_SA_TOKEN }} + kubectl config set-context my-context --cluster=my-cluster --user=github-actions --namespace=default + kubectl config use-context my-context + shell: bash + - name: Setup database port-forwarding + run: | + # Start port-forwarding for databases in background + kubectl port-forward -n ci-preview service/ci-preview-validator-1-service 5432:5432 & + kubectl port-forward -n sc service/gh-actions-postgres-service 5433:5432 & + + # Wait for port-forwards to be ready + sleep 10 + + # Verify port-forwards are working + nc -z localhost 5432 && echo "Port 5432 is ready" || echo "Port 5432 failed" + nc -z localhost 5433 && echo "Port 5433 is ready" || echo "Port 5433 failed" + shell: bash - name: Test against ci-preview uses: ./.github/actions/tests/run-e2e-tests with: env: ci decrypt: true blockchain: substrate + db_sync_host: localhost + db_sync_port: 5432 + db_host: localhost + db_port: 5433 post-merge-actions-complete: if: ${{ always() && (github.event_name == 'pull_request' && github.event.pull_request.merged == true) }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0e437857af..b7efb3c310 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -222,6 +222,32 @@ jobs: kubeconfig_base64: ${{ secrets.kubeconfig_base64 }} K8S_SERVER: ${{ secrets.K8S_SERVER }} K8S_SA_TOKEN: ${{ secrets.K8S_SA_TOKEN }} + - name: Install and configure kubectl + run: | + # Install kubectl + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x ./kubectl + sudo mv ./kubectl /usr/local/bin/kubectl + + # Configure kubectl + kubectl config set-cluster my-cluster --server=${{ secrets.K8S_SERVER }} --insecure-skip-tls-verify=true + kubectl config set-credentials github-actions --token=${{ secrets.K8S_SA_TOKEN }} + kubectl config set-context my-context --cluster=my-cluster --user=github-actions --namespace=default + kubectl config use-context my-context + shell: bash + - name: Setup database port-forwarding + run: | + # Start port-forwarding for databases in background + kubectl port-forward -n staging-preview service/staging-preview-validator-1-service 5432:5432 & + kubectl port-forward -n sc service/gh-actions-postgres-service 5433:5432 & + + # Wait for port-forwards to be ready + sleep 10 + + # Verify port-forwards are working + nc -z localhost 5432 && echo "Port 5432 is ready" || echo "Port 5432 failed" + nc -z localhost 5433 && echo "Port 5433 is ready" || echo "Port 5433 failed" + shell: bash - name: Run smoke tests uses: ./.github/actions/tests/run-e2e-tests with: @@ -230,6 +256,10 @@ jobs: decrypt: true markers: ${{ matrix.markers }} threads: 1 + db_sync_host: localhost + db_sync_port: 5432 + db_host: localhost + db_port: 5433 run-all-tests: if: always() && inputs.no-tests == false && needs.run-smoke-tests.result == 'success' @@ -250,6 +280,32 @@ jobs: kubeconfig_base64: ${{ secrets.kubeconfig_base64 }} K8S_SERVER: ${{ secrets.K8S_SERVER }} K8S_SA_TOKEN: ${{ secrets.K8S_SA_TOKEN }} + - name: Install and configure kubectl + run: | + # Install kubectl + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x ./kubectl + sudo mv ./kubectl /usr/local/bin/kubectl + + # Configure kubectl + kubectl config set-cluster my-cluster --server=${{ secrets.K8S_SERVER }} --insecure-skip-tls-verify=true + kubectl config set-credentials github-actions --token=${{ secrets.K8S_SA_TOKEN }} + kubectl config set-context my-context --cluster=my-cluster --user=github-actions --namespace=default + kubectl config use-context my-context + shell: bash + - name: Setup database port-forwarding + run: | + # Start port-forwarding for databases in background + kubectl port-forward -n staging-preview service/staging-preview-validator-1-service 5432:5432 & + kubectl port-forward -n sc service/gh-actions-postgres-service 5433:5432 & + + # Wait for port-forwards to be ready + sleep 10 + + # Verify port-forwards are working + nc -z localhost 5432 && echo "Port 5432 is ready" || echo "Port 5432 failed" + nc -z localhost 5433 && echo "Port 5433 is ready" || echo "Port 5433 failed" + shell: bash - name: Run all tests (some skipped due to new deployment) uses: ./.github/actions/tests/run-e2e-tests env: @@ -260,6 +316,10 @@ jobs: decrypt: true deployment_mc_epoch: $DEPLOYMENT_MC_EPOCH threads: 1 + db_sync_host: localhost + db_sync_port: 5432 + db_host: localhost + db_port: 5433 wait-for-n1-epoch: if: always() && inputs.no-tests == false && needs.partner-chain-ready.result == 'success' @@ -314,6 +374,32 @@ jobs: kubeconfig_base64: ${{ secrets.kubeconfig_base64 }} K8S_SERVER: ${{ secrets.K8S_SERVER }} K8S_SA_TOKEN: ${{ secrets.K8S_SA_TOKEN }} + - name: Install and configure kubectl + run: | + # Install kubectl + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x ./kubectl + sudo mv ./kubectl /usr/local/bin/kubectl + + # Configure kubectl + kubectl config set-cluster my-cluster --server=${{ secrets.K8S_SERVER }} --insecure-skip-tls-verify=true + kubectl config set-credentials github-actions --token=${{ secrets.K8S_SA_TOKEN }} + kubectl config set-context my-context --cluster=my-cluster --user=github-actions --namespace=default + kubectl config use-context my-context + shell: bash + - name: Setup database port-forwarding + run: | + # Start port-forwarding for databases in background + kubectl port-forward -n staging-preview service/staging-preview-validator-1-service 5432:5432 & + kubectl port-forward -n sc service/gh-actions-postgres-service 5433:5432 & + + # Wait for port-forwards to be ready + sleep 10 + + # Verify port-forwards are working + nc -z localhost 5432 && echo "Port 5432 is ready" || echo "Port 5432 failed" + nc -z localhost 5433 && echo "Port 5433 is ready" || echo "Port 5433 failed" + shell: bash - name: Run all tests (some skipped due to new deployment) uses: ./.github/actions/tests/run-e2e-tests env: @@ -325,6 +411,10 @@ jobs: latest_mc_epoch: true deployment_mc_epoch: $DEPLOYMENT_MC_EPOCH threads: 1 + db_sync_host: localhost + db_sync_port: 5432 + db_host: localhost + db_port: 5433 wait-for-n2-epoch: if: | @@ -385,6 +475,32 @@ jobs: kubeconfig_base64: ${{ secrets.kubeconfig_base64 }} K8S_SERVER: ${{ secrets.K8S_SERVER }} K8S_SA_TOKEN: ${{ secrets.K8S_SA_TOKEN }} + - name: Install and configure kubectl + run: | + # Install kubectl + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + chmod +x ./kubectl + sudo mv ./kubectl /usr/local/bin/kubectl + + # Configure kubectl + kubectl config set-cluster my-cluster --server=${{ secrets.K8S_SERVER }} --insecure-skip-tls-verify=true + kubectl config set-credentials github-actions --token=${{ secrets.K8S_SA_TOKEN }} + kubectl config set-context my-context --cluster=my-cluster --user=github-actions --namespace=default + kubectl config use-context my-context + shell: bash + - name: Setup database port-forwarding + run: | + # Start port-forwarding for databases in background + kubectl port-forward -n staging-preview service/staging-preview-validator-1-service 5432:5432 & + kubectl port-forward -n sc service/gh-actions-postgres-service 5433:5432 & + + # Wait for port-forwards to be ready + sleep 10 + + # Verify port-forwards are working + nc -z localhost 5432 && echo "Port 5432 is ready" || echo "Port 5432 failed" + nc -z localhost 5433 && echo "Port 5433 is ready" || echo "Port 5433 failed" + shell: bash - name: Run all tests (no skipped tests) uses: ./.github/actions/tests/run-e2e-tests env: @@ -396,6 +512,10 @@ jobs: latest_mc_epoch: true deployment_mc_epoch: $DEPLOYMENT_MC_EPOCH threads: 1 + db_sync_host: localhost + db_sync_port: 5432 + db_host: localhost + db_port: 5433 publish-ghcr-after-tests: if: | diff --git a/e2e-tests/tests/conftest.py b/e2e-tests/tests/conftest.py index 8fd84d9758..d1ef613f58 100644 --- a/e2e-tests/tests/conftest.py +++ b/e2e-tests/tests/conftest.py @@ -44,6 +44,12 @@ def pytest_addoption(parser): parser.addoption("--node-port", action="store", help="Overrides node port") parser.addoption("--deployment-mc-epoch", action="store", type=int, help="Deployment main chain epoch.") parser.addoption("--init-timestamp", action="store", type=int, help="Initial timestamp of the main chain.") + + # Database override options + parser.addoption("--db-host", action="store", help="Overrides qa_staging_db host") + parser.addoption("--db-port", action="store", help="Overrides qa_staging_db port") + parser.addoption("--db-sync-host", action="store", help="Overrides db-sync host") + parser.addoption("--db-sync-port", action="store", help="Overrides db-sync port") # committee tests parametrization parser.addoption( @@ -305,7 +311,7 @@ def config(): @fixture(scope="session") -def secrets(blockchain, nodes_env, decrypt, ci_run): +def secrets(request, blockchain, nodes_env, decrypt, ci_run): path = f"{os.getcwd()}/secrets/{blockchain}/{nodes_env}/{nodes_env}.json" assert os.path.isfile(path), f"Secrets file not found {path}" if decrypt: @@ -318,6 +324,31 @@ def secrets(blockchain, nodes_env, decrypt, ci_run): if ci_run and os.path.isfile(ci_path): secrets = secrets_ci(secrets, decrypt, ci_path) + # Apply command line database overrides + db_host = request.config.getoption("--db-host") + if db_host: + secrets["db"]["host"] = db_host + # Rebuild URL with new host + secrets["db"]["url"] = f"{secrets['db']['type']}://{secrets['db']['username']}:{secrets['db']['password']}@{db_host}:{secrets['db']['port']}/{secrets['db']['name']}" + + db_port = request.config.getoption("--db-port") + if db_port: + secrets["db"]["port"] = db_port + # Rebuild URL with new port + secrets["db"]["url"] = f"{secrets['db']['type']}://{secrets['db']['username']}:{secrets['db']['password']}@{secrets['db']['host']}:{db_port}/{secrets['db']['name']}" + + db_sync_host = request.config.getoption("--db-sync-host") + if db_sync_host: + secrets["dbSync"]["host"] = db_sync_host + # Rebuild URL with new host + secrets["dbSync"]["url"] = f"{secrets['dbSync']['type']}://{secrets['dbSync']['username']}:{secrets['dbSync']['password']}@{db_sync_host}:{secrets['dbSync']['port']}/{secrets['dbSync']['name']}" + + db_sync_port = request.config.getoption("--db-sync-port") + if db_sync_port: + secrets["dbSync"]["port"] = db_sync_port + # Rebuild URL with new port + secrets["dbSync"]["url"] = f"{secrets['dbSync']['type']}://{secrets['dbSync']['username']}:{secrets['dbSync']['password']}@{secrets['dbSync']['host']}:{db_sync_port}/{secrets['dbSync']['name']}" + return secrets