feat: add port-forwarding of db-sync and db-host to e2e-tests for automated ci-preview and staging-preview tests

.github/actions/tests/run-e2e-tests/action.yml

@@ -44,6 +44,18 @@ inputs:
     description: 'Number of threads to use'
     required: false
     default: '4'
+  db_host:
+    description: 'Override qa_staging_db host'
+    required: false
+  db_port:
+    description: 'Override qa_staging_db port'
+    required: false
+  db_sync_host:
+    description: 'Override db-sync host'
+    required: false
+  db_sync_port:
+    description: 'Override db-sync port'
+    required: false
 
 runs:
   using: composite
@@ -80,6 +92,27 @@ runs:
           decrypt_switch="--decrypt"
         fi
 
+        # Database override switches
+        db_host_switch=""
+        if [ -n "${{ inputs.db_host }}" ]; then
+          db_host_switch="--db-host '${{ inputs.db_host }}'"
+        fi
+
+        db_port_switch=""
+        if [ -n "${{ inputs.db_port }}" ]; then
+          db_port_switch="--db-port '${{ inputs.db_port }}'"
+        fi
+
+        db_sync_host_switch=""
+        if [ -n "${{ inputs.db_sync_host }}" ]; then
+          db_sync_host_switch="--db-sync-host '${{ inputs.db_sync_host }}'"
+        fi
+
+        db_sync_port_switch=""
+        if [ -n "${{ inputs.db_sync_port }}" ]; then
+          db_sync_port_switch="--db-sync-port '${{ inputs.db_sync_port }}'"
+        fi
+
         # Define pytest command
         pytest_cmd="pytest --blockchain '${{ inputs.blockchain }}' \
           --env '${{ inputs.env }}' \
@@ -93,6 +126,10 @@ runs:
           ${latest_mc_epoch_switch} \
           ${deployment_mc_epoch_switch} \
           ${decrypt_switch} \
+          ${db_host_switch} \
+          ${db_port_switch} \
+          ${db_sync_host_switch} \
+          ${db_sync_port_switch} \
           --json-report \
           --json-report-file=logs/.report.json \
           --json-report-summary \

.github/workflows/ci.yml

@@ -854,12 +854,42 @@ jobs:
           kubeconfig_base64: ${{ secrets.kubeconfig_base64 }}
           K8S_SERVER: ${{ secrets.K8S_SERVER }}
           K8S_SA_TOKEN: ${{ secrets.K8S_SA_TOKEN }}
+      - name: Install and configure kubectl
+        run: |
+          # Install kubectl
+          curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+          chmod +x ./kubectl
+          sudo mv ./kubectl /usr/local/bin/kubectl
+
+          # Configure kubectl
+          kubectl config set-cluster my-cluster --server=${{ secrets.K8S_SERVER }} --insecure-skip-tls-verify=true
+          kubectl config set-credentials github-actions --token=${{ secrets.K8S_SA_TOKEN }}
+          kubectl config set-context my-context --cluster=my-cluster --user=github-actions --namespace=default
+          kubectl config use-context my-context
+        shell: bash
+      - name: Setup database port-forwarding
+        run: |
+          # Start port-forwarding for databases in background
+          kubectl port-forward -n ci-preview service/ci-preview-validator-1-service 5432:5432 &
+          kubectl port-forward -n sc service/gh-actions-postgres-service 5433:5432 &
+
+          # Wait for port-forwards to be ready
+          sleep 10
+
+          # Verify port-forwards are working
+          nc -z localhost 5432 && echo "Port 5432 is ready" || echo "Port 5432 failed"
+          nc -z localhost 5433 && echo "Port 5433 is ready" || echo "Port 5433 failed"
+        shell: bash
       - name: Test against ci-preview
         uses: ./.github/actions/tests/run-e2e-tests
         with:
           env: ci
           decrypt: true
           blockchain: substrate
+          db_sync_host: localhost
+          db_sync_port: 5432
+          db_host: localhost
+          db_port: 5433
 
   post-merge-actions-complete:
     if: ${{ always() && (github.event_name == 'pull_request' && github.event.pull_request.merged == true) }}

.github/workflows/release.yml

@@ -222,6 +222,32 @@ jobs:
           kubeconfig_base64: ${{ secrets.kubeconfig_base64 }}
           K8S_SERVER: ${{ secrets.K8S_SERVER }}
           K8S_SA_TOKEN: ${{ secrets.K8S_SA_TOKEN }}
+      - name: Install and configure kubectl
+        run: |
+          # Install kubectl
+          curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+          chmod +x ./kubectl
+          sudo mv ./kubectl /usr/local/bin/kubectl
+
+          # Configure kubectl
+          kubectl config set-cluster my-cluster --server=${{ secrets.K8S_SERVER }} --insecure-skip-tls-verify=true
+          kubectl config set-credentials github-actions --token=${{ secrets.K8S_SA_TOKEN }}
+          kubectl config set-context my-context --cluster=my-cluster --user=github-actions --namespace=default
+          kubectl config use-context my-context
+        shell: bash
+      - name: Setup database port-forwarding
+        run: |
+          # Start port-forwarding for databases in background
+          kubectl port-forward -n staging-preview service/staging-preview-validator-1-service 5432:5432 &
+          kubectl port-forward -n sc service/gh-actions-postgres-service 5433:5432 &
+
+          # Wait for port-forwards to be ready
+          sleep 10
+
+          # Verify port-forwards are working
+          nc -z localhost 5432 && echo "Port 5432 is ready" || echo "Port 5432 failed"
+          nc -z localhost 5433 && echo "Port 5433 is ready" || echo "Port 5433 failed"
+        shell: bash
       - name: Run smoke tests
         uses: ./.github/actions/tests/run-e2e-tests
         with:
@@ -230,6 +256,10 @@ jobs:
           decrypt: true
           markers: ${{ matrix.markers }}
           threads: 1
+          db_sync_host: localhost
+          db_sync_port: 5432
+          db_host: localhost
+          db_port: 5433
 
   run-all-tests:
     if: always() && inputs.no-tests == false && needs.run-smoke-tests.result == 'success'
@@ -250,6 +280,32 @@ jobs:
           kubeconfig_base64: ${{ secrets.kubeconfig_base64 }}
           K8S_SERVER: ${{ secrets.K8S_SERVER }}
           K8S_SA_TOKEN: ${{ secrets.K8S_SA_TOKEN }}
+      - name: Install and configure kubectl
+        run: |
+          # Install kubectl
+          curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+          chmod +x ./kubectl
+          sudo mv ./kubectl /usr/local/bin/kubectl
+
+          # Configure kubectl
+          kubectl config set-cluster my-cluster --server=${{ secrets.K8S_SERVER }} --insecure-skip-tls-verify=true
+          kubectl config set-credentials github-actions --token=${{ secrets.K8S_SA_TOKEN }}
+          kubectl config set-context my-context --cluster=my-cluster --user=github-actions --namespace=default
+          kubectl config use-context my-context
+        shell: bash
+      - name: Setup database port-forwarding
+        run: |
+          # Start port-forwarding for databases in background
+          kubectl port-forward -n staging-preview service/staging-preview-validator-1-service 5432:5432 &
+          kubectl port-forward -n sc service/gh-actions-postgres-service 5433:5432 &
+
+          # Wait for port-forwards to be ready
+          sleep 10
+
+          # Verify port-forwards are working
+          nc -z localhost 5432 && echo "Port 5432 is ready" || echo "Port 5432 failed"
+          nc -z localhost 5433 && echo "Port 5433 is ready" || echo "Port 5433 failed"
+        shell: bash
       - name: Run all tests (some skipped due to new deployment)
         uses: ./.github/actions/tests/run-e2e-tests
         env:
@@ -260,6 +316,10 @@ jobs:
           decrypt: true
           deployment_mc_epoch: $DEPLOYMENT_MC_EPOCH
           threads: 1
+          db_sync_host: localhost
+          db_sync_port: 5432
+          db_host: localhost
+          db_port: 5433
 
   wait-for-n1-epoch:
     if: always() && inputs.no-tests == false && needs.partner-chain-ready.result == 'success'
@@ -314,6 +374,32 @@ jobs:
           kubeconfig_base64: ${{ secrets.kubeconfig_base64 }}
           K8S_SERVER: ${{ secrets.K8S_SERVER }}
           K8S_SA_TOKEN: ${{ secrets.K8S_SA_TOKEN }}
+      - name: Install and configure kubectl
+        run: |
+          # Install kubectl
+          curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+          chmod +x ./kubectl
+          sudo mv ./kubectl /usr/local/bin/kubectl
+
+          # Configure kubectl
+          kubectl config set-cluster my-cluster --server=${{ secrets.K8S_SERVER }} --insecure-skip-tls-verify=true
+          kubectl config set-credentials github-actions --token=${{ secrets.K8S_SA_TOKEN }}
+          kubectl config set-context my-context --cluster=my-cluster --user=github-actions --namespace=default
+          kubectl config use-context my-context
+        shell: bash
+      - name: Setup database port-forwarding
+        run: |
+          # Start port-forwarding for databases in background
+          kubectl port-forward -n staging-preview service/staging-preview-validator-1-service 5432:5432 &
+          kubectl port-forward -n sc service/gh-actions-postgres-service 5433:5432 &
+
+          # Wait for port-forwards to be ready
+          sleep 10
+
+          # Verify port-forwards are working
+          nc -z localhost 5432 && echo "Port 5432 is ready" || echo "Port 5432 failed"
+          nc -z localhost 5433 && echo "Port 5433 is ready" || echo "Port 5433 failed"
+        shell: bash
       - name: Run all tests (some skipped due to new deployment)
         uses: ./.github/actions/tests/run-e2e-tests
         env:
@@ -325,6 +411,10 @@ jobs:
           latest_mc_epoch: true
           deployment_mc_epoch: $DEPLOYMENT_MC_EPOCH
           threads: 1
+          db_sync_host: localhost
+          db_sync_port: 5432
+          db_host: localhost
+          db_port: 5433
 
   wait-for-n2-epoch:
     if: |
@@ -385,6 +475,32 @@ jobs:
           kubeconfig_base64: ${{ secrets.kubeconfig_base64 }}
           K8S_SERVER: ${{ secrets.K8S_SERVER }}
           K8S_SA_TOKEN: ${{ secrets.K8S_SA_TOKEN }}
+      - name: Install and configure kubectl
+        run: |
+          # Install kubectl
+          curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+          chmod +x ./kubectl
+          sudo mv ./kubectl /usr/local/bin/kubectl
+
+          # Configure kubectl
+          kubectl config set-cluster my-cluster --server=${{ secrets.K8S_SERVER }} --insecure-skip-tls-verify=true
+          kubectl config set-credentials github-actions --token=${{ secrets.K8S_SA_TOKEN }}
+          kubectl config set-context my-context --cluster=my-cluster --user=github-actions --namespace=default
+          kubectl config use-context my-context
+        shell: bash
+      - name: Setup database port-forwarding
+        run: |
+          # Start port-forwarding for databases in background
+          kubectl port-forward -n staging-preview service/staging-preview-validator-1-service 5432:5432 &
+          kubectl port-forward -n sc service/gh-actions-postgres-service 5433:5432 &
+
+          # Wait for port-forwards to be ready
+          sleep 10
+
+          # Verify port-forwards are working
+          nc -z localhost 5432 && echo "Port 5432 is ready" || echo "Port 5432 failed"
+          nc -z localhost 5433 && echo "Port 5433 is ready" || echo "Port 5433 failed"
+        shell: bash
       - name: Run all tests (no skipped tests)
         uses: ./.github/actions/tests/run-e2e-tests
         env:
@@ -396,6 +512,10 @@ jobs:
           latest_mc_epoch: true
           deployment_mc_epoch: $DEPLOYMENT_MC_EPOCH
           threads: 1
+          db_sync_host: localhost
+          db_sync_port: 5432
+          db_host: localhost
+          db_port: 5433
 
   publish-ghcr-after-tests:
     if: |

e2e-tests/tests/conftest.py

@@ -44,6 +44,12 @@ def pytest_addoption(parser):
     parser.addoption("--node-port", action="store", help="Overrides node port")
     parser.addoption("--deployment-mc-epoch", action="store", type=int, help="Deployment main chain epoch.")
     parser.addoption("--init-timestamp", action="store", type=int, help="Initial timestamp of the main chain.")
+
+    # Database override options
+    parser.addoption("--db-host", action="store", help="Overrides qa_staging_db host")
+    parser.addoption("--db-port", action="store", help="Overrides qa_staging_db port")
+    parser.addoption("--db-sync-host", action="store", help="Overrides db-sync host")
+    parser.addoption("--db-sync-port", action="store", help="Overrides db-sync port")
 
     # committee tests parametrization
     parser.addoption(
@@ -305,7 +311,7 @@ def config():
 
 
 @fixture(scope="session")
-def secrets(blockchain, nodes_env, decrypt, ci_run):
+def secrets(request, blockchain, nodes_env, decrypt, ci_run):
     path = f"{os.getcwd()}/secrets/{blockchain}/{nodes_env}/{nodes_env}.json"
     assert os.path.isfile(path), f"Secrets file not found {path}"
     if decrypt:
@@ -318,6 +324,31 @@ def secrets(blockchain, nodes_env, decrypt, ci_run):
     if ci_run and os.path.isfile(ci_path):
         secrets = secrets_ci(secrets, decrypt, ci_path)
 
+    # Apply command line database overrides
+    db_host = request.config.getoption("--db-host")
+    if db_host:
+        secrets["db"]["host"] = db_host
+        # Rebuild URL with new host
+        secrets["db"]["url"] = f"{secrets['db']['type']}://{secrets['db']['username']}:{secrets['db']['password']}@{db_host}:{secrets['db']['port']}/{secrets['db']['name']}"
+
+    db_port = request.config.getoption("--db-port") 
+    if db_port:
+        secrets["db"]["port"] = db_port
+        # Rebuild URL with new port
+        secrets["db"]["url"] = f"{secrets['db']['type']}://{secrets['db']['username']}:{secrets['db']['password']}@{secrets['db']['host']}:{db_port}/{secrets['db']['name']}"
+
+    db_sync_host = request.config.getoption("--db-sync-host")
+    if db_sync_host:
+        secrets["dbSync"]["host"] = db_sync_host
+        # Rebuild URL with new host
+        secrets["dbSync"]["url"] = f"{secrets['dbSync']['type']}://{secrets['dbSync']['username']}:{secrets['dbSync']['password']}@{db_sync_host}:{secrets['dbSync']['port']}/{secrets['dbSync']['name']}"
+
+    db_sync_port = request.config.getoption("--db-sync-port")
+    if db_sync_port:
+        secrets["dbSync"]["port"] = db_sync_port
+        # Rebuild URL with new port  
+        secrets["dbSync"]["url"] = f"{secrets['dbSync']['type']}://{secrets['dbSync']['username']}:{secrets['dbSync']['password']}@{secrets['dbSync']['host']}:{db_sync_port}/{secrets['dbSync']['name']}"
+
     return secrets