(PE-40222) add CI test for replacing a failed postgres server

.github/workflows/test-replace-failed-postgresql.yaml

@@ -0,0 +1,131 @@
+---
+name: Test replace failed PostgreSQL
+on:
+  pull_request:
+    paths:
+      - .github/workflows/**/*
+      - spec/**/*
+      - lib/**/*
+      - tasks/**/*
+      - functions/**/*
+      - types/**/*
+      - plans/**/*
+      - hiera/**/*
+      - manifests/**/*
+      - templates/**/*
+      - files/**/*
+      - metadata.json
+      - Rakefile
+      - Gemfile
+      - provision.yaml
+      - .rspec
+      - .rubocop.yml
+      - .puppet-lint.rc
+      - .fixtures.yml
+    branches: [main]
+  workflow_dispatch: {}
+jobs:
+  test-replace-failed-postgresql:
+    name: PE ${{ matrix.version }} ${{ matrix.architecture }} on ${{ matrix.image }}
+    runs-on: ubuntu-20.04
+    env:
+      BOLT_GEM: true
+      BOLT_DISABLE_ANALYTICS: true
+      LANG: en_US.UTF-8
+    strategy:
+      fail-fast: false
+      matrix:
+        architecture: [extra-large-with-dr-and-spare-replica]
+        install_architecture: [extra-large-with-dr]
+        version: [2021.7.9, 2023.8.1, 2025.0.0]
+        image: [almalinux-cloud/almalinux-8]
+    steps:
+      - name: Checkout Source
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+      - name: Activate Ruby 2.7
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '2.7'
+          bundler-cache: true
+      - name: Print bundle environment
+        if: ${{ github.repository_owner == 'puppetlabs' }}
+        run: |
+          echo ::group::info:bundler
+            bundle env
+          echo ::endgroup::
+      - name: Provision test cluster
+        timeout-minutes: 15
+        run: |
+          echo ::group::prepare
+            mkdir -p $HOME/.ssh
+            echo 'Host *'                      >  $HOME/.ssh/config
+            echo '    ServerAliveInterval 150' >> $HOME/.ssh/config
+            echo '    ServerAliveCountMax 2'   >> $HOME/.ssh/config
+            bundle exec rake spec_prep
+          echo ::endgroup::
+          echo ::group::provision
+            bundle exec bolt plan run peadm_spec::provision_test_cluster \
+              --modulepath spec/fixtures/modules \
+              provider=provision_service \
+              image=${{ matrix.image }} \
+              architecture=${{ matrix.architecture }} \
+              --log-level trace
+          echo ::endgroup::
+          echo ::group::info:request
+            cat request.json || true; echo
+          echo ::endgroup::
+          echo ::group::info:inventory
+            sed -e 's/password: .*/password: "[redacted]"/' < spec/fixtures/litmus_inventory.yaml || true
+          echo ::endgroup::
+          echo ::group::certnames
+            bundle exec bolt plan run peadm_spec::add_inventory_hostnames \
+            --inventory spec/fixtures/litmus_inventory.yaml \
+            --modulepath spec/fixtures/modules \
+            --no-host-key-check \
+            inventory_file=spec/fixtures/litmus_inventory.yaml
+          echo ::endgroup::            
+      - name: Output contents of litmus_inventory.yaml
+        run: |
+          cat spec/fixtures/litmus_inventory.yaml
+      - name: Install PE on test cluster
+        timeout-minutes: 120
+        run: |
+          bundle exec bolt plan run peadm_spec::install_test_cluster \
+            --inventoryfile spec/fixtures/litmus_inventory.yaml \
+            --modulepath spec/fixtures/modules \
+            architecture=${{ matrix.install_architecture }} \
+            version=${{ matrix.version }} \
+            console_password=${{ secrets.CONSOLE_PASSWORD }} \
+            code_manager_auto_configure=true
+      - name: Replace failed PostgreSQL
+        run: |
+          echo ::group::prepare
+            mkdir -p $HOME/.ssh
+            echo 'Host *'                      >  $HOME/.ssh/config
+            echo '    ServerAliveInterval 150' >> $HOME/.ssh/config
+            echo '    ServerAliveCountMax 2'   >> $HOME/.ssh/config
+            bundle exec rake spec_prep
+          echo ::endgroup::
+          bundle exec bolt plan run peadm_spec::test_replace_failed_postgres \
+            --inventoryfile spec/fixtures/litmus_inventory.yaml \
+            --modulepath spec/fixtures/modules \
+            primary_host=$(yq '.groups[].targets[] | select(.vars.role == "primary") | .name' spec/fixtures/litmus_inventory.yaml) \
+            replica_host=$(yq '.groups[].targets[] | select(.vars.role == "replica") | .name' spec/fixtures/litmus_inventory.yaml) \
+            working_postgresql_host=$(yq '.groups[].targets[] | select(.vars.role == "replica-pdb-postgresql") | .name' spec/fixtures/litmus_inventory.yaml) \
+            failed_postgresql_host=$(yq '.groups[].targets[] | select(.vars.role == "primary-pdb-postgresql") | .name' spec/fixtures/litmus_inventory.yaml) \
+            replacement_postgresql_host=$(yq '.groups[].targets[] | select(.vars.role == "spare-replica") | .name' spec/fixtures/litmus_inventory.yaml) \
+          --no-host-key-check
+      - name: Tear down PE XL DR test cluster
+        if: ${{ always() }}
+        continue-on-error: true
+        run: |-
+          if [ -f spec/fixtures/litmus_inventory.yaml ]; then
+            echo ::group::tear_down
+              bundle exec rake 'litmus:tear_down'
+            echo ::endgroup::
+            echo ::group::info:request
+              cat request.json || true; echo
+            echo ::endgroup::
+          fi

plans/replace_failed_postgresql.pp

@@ -27,9 +27,9 @@
   $peadm_config = run_task('peadm::get_peadm_config', $primary_host).first.value
   $compilers = $peadm_config['params']['compilers']
 
-  # Bail if this is trying to be ran against Standard
+  # Bail if we are not running this against an XL deployment with DR enabled - the parameters also enforce this to some extent
   if $compilers.empty {
-    fail_plan('Plan peadm::replace_failed_postgresql is only applicable for L and XL deployments')
+    fail_plan('Plan peadm::replace_failed_postgresql is only applicable for XL deployments with DR enabled')
   }
 
   $pe_hosts = peadm::flatten_compact([

spec/acceptance/peadm_spec/plans/test_replace_failed_postgres.pp

@@ -0,0 +1,43 @@
+plan peadm_spec::test_replace_failed_postgres(
+  Peadm::SingleTargetSpec   $primary_host,
+  Peadm::SingleTargetSpec   $replica_host,
+  Peadm::SingleTargetSpec   $working_postgresql_host,
+  Peadm::SingleTargetSpec   $failed_postgresql_host,
+  Peadm::SingleTargetSpec   $replacement_postgresql_host,
+) {
+  # run infra status on the primary
+  out::message("Running peadm::status on primary host ${primary_host}")
+  $primary_status = run_plan('peadm::status', $primary_host, { 'format' => 'json' })
+  out::message($primary_status)
+
+  if empty($primary_status['failed']) {
+    out::message('Cluster is healthy, continuing')
+  } else {
+    fail_plan('Cluster is not healthy, aborting')
+  }
+
+  # replace the failed postgres server
+  run_plan('peadm::replace_failed_postgresql',
+    primary_host => $primary_host,
+    replica_host => $replica_host,
+    working_postgresql_host => $working_postgresql_host,
+    failed_postgresql_host => $failed_postgresql_host,
+    replacement_postgresql_host => $replacement_postgresql_host,
+  )
+
+  # get the config from primary_host and verify failed_postgresql_host is removed and replacement was added
+  $result = run_task('peadm::get_peadm_config', $primary_host, '_catch_errors' => true).first.to_data()
+  $primary_postgres_host = $result['value']['params']['primary_postgresql_host']
+  $replica_postgres_host = $result['value']['params']['replica_postgresql_host']
+
+  if $primary_postgres_host == $failed_postgresql_host or $replica_postgres_host == $failed_postgresql_host {
+    fail_plan("Failed PostgreSQL host ${failed_postgresql_host} was not removed from the PE configuration")
+  } else {
+    out::message("Failed PostgreSQL host ${failed_postgresql_host} was removed from the PE configuration")
+  }
+  if $primary_postgres_host == $replacement_postgresql_host or $replica_postgres_host == $replacement_postgresql_host {
+    out::message("Replacement PostgreSQL host ${replacement_postgresql_host} was added to the PE configuration")
+  } else {
+    fail_plan("Replacement PostgreSQL host ${replacement_postgresql_host} was not added the PE configuration")
+  }
+}

spec/docker/bolt-project.yaml

@@ -2,12 +2,18 @@
 name: peadm_docker_examples
 modules:
   - name: nwops/container_inventory
-    version_requirement: ">= 0.1.1"
+    version_requirement: '>= 0.1.1'
   - name: puppetlabs/stdlib
-    version_requirement: ">= 6.5.0 < 8.0.0"
+    version_requirement: '>= 9.0.0 < 10.0.0'
   - puppetlabs/ruby_task_helper
   - name: puppetlabs/node_manager
-    version_requirement: ">= 1.0.1 < 2.0.0"
+    version_requirement: '>= 1.0.1 < 2.0.0'
+  - name: puppetlabs/service
+    version_requirement: '>= 1.3.0 <= 4.0.0'
+  - name: puppetlabs/package
+    version_requirement: '>= 2.1.0 <= 4.0.0'
+  - name: puppetlabs/inifile
+    version_requirement: '>= 6.1.0 < 7.0.0'
   - puppetlabs/apply_helpers
   - puppetlabs/bolt_shim
   - puppet/format