functional-tests.yml

name: Functional tests

# NOTE(mikal): git repos are checked out to /srv/github/_work/{repo}/{repo}
# which is available as GITHUB_WORKSPACE. You can find other environment
# variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables

on:
  pull_request:
    branches:
      - develop
      - v*-releases
    paths-ignore:
      - 'docs/**'

jobs:
  lint:
    runs-on: self-hosted
    concurrency:
      group: ${{ github.workflow }}-${{ github.ref }}-lint
      cancel-in-progress: true

    steps:
      - name: Checkout code with two commits
        uses: actions/checkout@v4
        with:
          fetch-depth: 2

      - name: Lint with flake8
        run: |
          /usr/bin/tox -eflake8

  # NOTE(mikal): we need this one early because renovate creates a lot of PRs
  # where the proposed combination of dependencies doesn't actually install and
  # I don't want to waste a lot of CI resources on them.
  requirements-check:
    needs: lint
    runs-on: self-hosted
    concurrency:
      group: ${{ github.workflow }}-${{ github.ref }}-requirements
      cancel-in-progress: true

    steps:
      - name: Checkout shakenfist
        uses: actions/checkout@v4
        with:
          path: shakenfist
          fetch-depth: 0

      - name: Attempt to install requirements
        run: |
          cd ${GITHUB_WORKSPACE}
          python3 -mvenv venv
          venv/bin/pip3 install -r shakenfist/requirements.txt

  build38:
    needs: lint
    runs-on: self-hosted
    concurrency:
      group: ${{ github.workflow }}-${{ github.ref }}-build38
      cancel-in-progress: true

    steps:
      - name: Checkout code with two commits
        uses: actions/checkout@v4
        with:
          fetch-depth: 2

      - name: Run python3 unit tests
        run: |
          /usr/bin/tox -epy3

  functional_matrix:
    needs: requirements-check
    strategy:
      fail-fast: false
      matrix:
        os: [
          {
            job_name: 'debian-11-localhost',
            base_image: 'sf://label/ci-images/debian-11',
            base_image_user: 'debian',
            topology: 'localhost',
            concurrency: 3
          },
          {
            job_name: 'debian-12-slim-primary',
            base_image: 'sf://label/ci-images/debian-12',
            base_image_user: 'debian',
            topology: 'slim-primary',
            concurrency: 5
          },
          {
            job_name: 'ubuntu-2004-localhost',
            base_image: 'sf://label/ci-images/ubuntu-2004',
            base_image_user: 'ubuntu',
            topology: 'localhost',
            concurrency: 3
          },
          {
            job_name: 'ubuntu-2204-slim-primary',
            base_image: 'sf://label/ci-images/ubuntu-2204',
            base_image_user: 'ubuntu',
            topology: 'slim-primary',
            concurrency: 5
          },
        ]
    runs-on: self-hosted
    concurrency:
      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os.job_name }}
      cancel-in-progress: true

    steps:
      - name: Log matrix details
        run: |
          echo "job_name: ${{ matrix.os.job_name }}"
          echo "base_image: ${{ matrix.os.base_image }}"
          echo "base_image_user: ${{ matrix.os.base_image_user }}"
          echo "topology: ${{ matrix.os.topology }}"
          echo "concurrency: ${{ matrix.os.concurrency }}"

      - name: Set environment variables
        run: |
          echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
          echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV
          echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV

      - name: Checkout shakenfist
        uses: actions/checkout@v4
        with:
          path: shakenfist
          fetch-depth: 0

      - name: Determine if there is any dependency between the repositories
        run: |
          python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py

      - name: Build infrastructure
        run: |
          cd ${GITHUB_WORKSPACE}/shakenfist
          ansible-playbook -i /home/debian/ansible-hosts \
              --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \
              base_image=${{ matrix.os.base_image }} base_image_user=${{ matrix.os.base_image_user }}" \
              deploy/ansible/ci-topology-${{ matrix.os.topology }}.yml

      - name: Copy CI tools to primary
        run: |
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          cd ${GITHUB_WORKSPACE}/shakenfist
          scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null -rp tools \
              ${{ matrix.os.base_image_user }}@${primary}:.

      - name: Log github actions buffering status
        run: |
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          cd ${GITHUB_WORKSPACE}/shakenfist
          tools/run_remote ${primary} python3 tools/buffer.py

      - name: Run getsf installer on primary
        run: |
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              ${{ matrix.os.base_image_user }}@${primary} /tmp/getsf-wrapper
          echo ""
          echo ""
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              ${{ matrix.os.base_image_user }}@${primary} \
              'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful'

      - name: Wait for API to start answering
        run: |
          set +e

          . ${GITHUB_WORKSPACE}/ci-environment.sh
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              ${{ matrix.os.base_image_user }}@${primary} 'sudo chmod ugo+r /etc/sf/* /var/log/syslog'

          count=0
          while [ $count -lt 60 ]
          do
            ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              ${{ matrix.os.base_image_user }}@${primary} '. /etc/sf/sfrc; sf-client instance list'
            if [ $? == 0 ]; then
              exit 0
            fi

            count=$(( $count + 1 ))
            sleep 5
          done

          exit 1

      - name: Import cached images
        run: |
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              ${{ matrix.os.base_image_user }}@${primary} \
              '. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared'
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              ${{ matrix.os.base_image_user }}@${primary} \
              '. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared'
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              ${{ matrix.os.base_image_user }}@${primary} \
              '. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared'
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              ${{ matrix.os.base_image_user }}@${primary} \
              '. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared'

      - name: Create a base level of activity in the cluster
        run: |
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              ${{ matrix.os.base_image_user }}@${primary} \
              'echo "==== sfrc ===="; cat /etc/sf/sfrc; echo "==== end sfrc ===="'
          echo ""
          echo ""
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              ${{ matrix.os.base_image_user }}@${primary} \
              '. /etc/sf/sfrc; for i in `seq 100`; do sf-client --async=continue network create background-$i 10.$i.0.0/24 > /dev/null; echo -n "."; done'
          echo ""

      - name: Run functional tests
        timeout-minutes: 120
        run: |
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              $source_path/shakenfist \
              ${{ matrix.os.base_image_user }}@${primary}:shakenfist
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              ${{ matrix.os.base_image_user }}@${primary} "cd shakenfist/deploy; . /etc/sf/sfrc; sudo PIP_BREAK_SYSTEM_PACKAGES=1 pip3 install -r requirements.txt; set -e; stestr run --concurrency=${{ matrix.os.concurrency}}; stestr slowest"

      - name: Check logs
        if: always()
        run: |
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          cd ${GITHUB_WORKSPACE}/shakenfist
          tools/run_remote ${primary} sudo /home/${{ matrix.os.base_image_user }}/tools/ci_event_checks.sh

      # On Ubuntu 22.04 the cleaner is rated a CPU hog because of etcd cleanup
      # cost. That's not really something we can control, so just ignore the CPU
      # usage of that process instead.
      - name: Check SF process CPU usage
        if: always()
        run: |
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              ${{ matrix.os.base_image_user }}@${primary} '. /etc/sf/sfrc; sf-client node cpuhogs --ignore sf_cleaner'

      - name: Check for reasonable data rates
        timeout-minutes: 5
        if: always()
        run: |
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          cd ${GITHUB_WORKSPACE}/shakenfist
          tools/run_remote ${primary} sudo /home/${{ matrix.os.base_image_user }}/tools/ci_event_checks.sh

      - name: Fetch and tweak inventory
        if: always()
        run: |
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              ${{ matrix.os.base_image_user }}@${primary}:/etc/sf/inventory.yaml /srv/github/
          sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml

          echo "====="
          cat /srv/github/inventory.yaml

      - name: Gather logs
        if: always()
        run: |
          set -x

          # We need the ssh key in the place ansible expects it to be, which isn't
          # true on the CI worker node.
          cp /srv/github/id_ci /home/debian/.ssh/id_rsa
          cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub

          ansible-playbook -i /srv/github/inventory.yaml \
              --extra-vars "base_image_user=${{ matrix.os.base_image_user }} \
              ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \
              ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml

      - uses: actions/upload-artifact@v4
        if: always()
        with:
          name: bundle-${{ matrix.os.job_name }}.zip
          retention-days: 90
          if-no-files-found: error
          path: /srv/github/artifacts/bundle.zip

  ansible-modules:
    needs: requirements-check
    runs-on: self-hosted
    concurrency:
      group: ${{ github.workflow }}-${{ github.ref }}-ansiblemodules
      cancel-in-progress: true

    steps:
      - name: Set environment variables
        run: |
          echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
          echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV
          echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV

      - name: Checkout shakenfist
        uses: actions/checkout@v4
        with:
          path: shakenfist
          fetch-depth: 0

      - name: Determine if there is any dependency between the repositories
        run: |
          python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py

      - name: Build infrastructure
        run: |
          cd ${GITHUB_WORKSPACE}/shakenfist
          ansible-playbook -i /home/debian/ansible-hosts \
              --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \
              base_image=sf://label/ci-images/debian-11 base_image_user=debian" \
              deploy/ansible/ci-topology-slim-primary.yml

      - name: Run getsf installer on primary
        run: |
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              debian@$primary /tmp/getsf-wrapper
          echo ""
          echo ""
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              debian@$primary \
              'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful'

      - name: Wait for API to start answering
        run: |
          set +e

          . ${GITHUB_WORKSPACE}/ci-environment.sh
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              debian@$primary 'sudo chmod ugo+r /etc/sf/* /var/log/syslog'

          count=0
          while [ $count -lt 60 ]
          do
            ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              debian@$primary '. /etc/sf/sfrc; sf-client instance list'
            if [ $? == 0 ]; then
              exit 0
            fi

            count=$(( $count + 1 ))
            sleep 5
          done

          exit 1

      - name: Import cached images
        run: |
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              debian@$primary \
              '. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared'
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              debian@$primary \
              '. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared'
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              debian@$primary \
              '. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared'
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              debian@$primary \
              '. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared'

      - name: Run ansible module tests
        timeout-minutes: 120
        run: |
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              $source_path/shakenfist \
              debian@$primary:shakenfist
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              debian@$primary "cd shakenfist/deploy; . /etc/sf/sfrc; bash ansiblemoduletests.sh"

      - name: Check logs
        if: always()
        run: |
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
                -o UserKnownHostsFile=/dev/null \
                debian@$primary sudo chmod ugo+r /var/log/syslog
          scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              debian@$primary:/var/log/syslog \
              ${GITHUB_WORKSPACE}/syslog

          failures=0

          echo
          etcd_conns=`grep -c "Building new etcd connection" ${GITHUB_WORKSPACE}/syslog || true`
          echo "This CI run created $etcd_conns etcd connections."
          if [ $etcd_conns -gt 5000 ]; then
            echo "FAILURE: Too many etcd clients!"
            failures=1
          fi

          echo
          sigterms=`grep -c "Sent SIGTERM to " ${GITHUB_WORKSPACE}/syslog || true`
          echo "This CI run sent $sigterms SIGTERM signals while shutting down."
          if [ $sigterms -gt 50 ]; then
            echo "FAILURE: Too many SIGTERMs sent!"
            failures=1
          fi

          FORBIDDEN=("Traceback (most recent call last):"
                     "ERROR sf"
                     "ERROR gunicorn"
                     " died"
                     "Extra vxlan present"
                     "Fork support is only compatible with the epoll1 and poll polling strategies"
                     "not using configured address"
                     "Dumping thread traces"
                     "because it is leased to"
                     "not committing online upgrade"
                     "Received a GOAWAY with error code ENHANCE_YOUR_CALM"
                     "ConnectionFailedError"
                     "invalid JWT in Authorization header"
                     "Libvirt Error: XML error"
                     "Cleaning up leaked IPAM"
                     "Cleaning up leaked vxlan"
                     "Waiting to acquire lock"
                     'apparmor="DENIED"'
                     "Ignoring malformed cache entry")
          IFS=""
          for forbid in ${FORBIDDEN[*]}
          do
            if [ `grep -c "$forbid" ${GITHUB_WORKSPACE}/syslog || true` -gt 0 ]
            then
              echo "FAILURE: Forbidden string found in logs: $forbid"
              failures=1
            fi
          done

          if [ $failures -gt 0 ]; then
              echo "...failures detected."
              exit 1
          fi

      - name: Check process CPU usage
        if: always()
        run: |
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              debian@$primary '. /etc/sf/sfrc; sf-client node cpuhogs'

      - name: Fetch and tweak inventory
        if: always()
        run: |
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              debian@$primary:/etc/sf/inventory.yaml /srv/github/
          sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml

          echo "====="
          cat /srv/github/inventory.yaml

      - name: Gather logs
        if: always()
        run: |
          set -x

          # We need the ssh key in the place ansible expects it to be, which isn't
          # true on the CI worker node.
          cp /srv/github/id_ci /home/debian/.ssh/id_rsa
          cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub
          chown -R debian.debian /home/debian/.ssh

          ansible-playbook -i /srv/github/inventory.yaml \
              --extra-vars "base_image_user=debian ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \
              ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml

      - uses: actions/upload-artifact@v4
        if: always()
        with:
          name: bundle-ansible-modules.zip
          retention-days: 90
          if-no-files-found: error
          path: /srv/github/artifacts/bundle.zip


  node-lifecycle:
    needs: requirements-check
    runs-on: self-hosted
    concurrency:
      group: ${{ github.workflow }}-${{ github.ref }}-nodelifecycle
      cancel-in-progress: true

    steps:
      - name: Set environment variables
        run: |
          echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
          echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV
          echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV

      - name: Checkout shakenfist
        uses: actions/checkout@v4
        with:
          path: shakenfist
          fetch-depth: 0

      - name: Determine if there is any dependency between the repositories
        run: |
          python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py

      - name: Build infrastructure
        run: |
          cd ${GITHUB_WORKSPACE}/shakenfist
          ansible-playbook -i /home/debian/ansible-hosts \
              --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \
              base_image=sf://label/ci-images/debian-11 base_image_user=debian" \
              deploy/ansible/ci-topology-slim-primary.yml

      - name: Copy CI tools to primary
        run: |
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          cd ${GITHUB_WORKSPACE}/shakenfist
          scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null -rp tools \
              debian@$primary:.

      - name: Run getsf installer on primary
        run: |
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              debian@$primary /tmp/getsf-wrapper
          echo ""
          echo ""
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              debian@$primary \
              'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful'

      - name: Wait for API to start answering
        run: |
          set +e

          . ${GITHUB_WORKSPACE}/ci-environment.sh
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              debian@$primary 'sudo chmod ugo+r /etc/sf/* /var/log/syslog'

          count=0
          while [ $count -lt 60 ]
          do
            ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              debian@$primary '. /etc/sf/sfrc; sf-client instance list'
            if [ $? == 0 ]; then
              exit 0
            fi

            count=$(( $count + 1 ))
            sleep 5
          done

          exit 1

      - name: Import cached images
        run: |
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              debian@$primary \
              '. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared'
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              debian@$primary \
              '. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared'
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              debian@$primary \
              '. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared'
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              debian@$primary \
              '. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared'

      - name: Run node lifecycle tests
        timeout-minutes: 120
        run: |
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              $source_path/shakenfist \
              debian@$primary:shakenfist
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              debian@$primary "cd shakenfist/deploy; . /etc/sf/sfrc; bash nodelifecycletests.sh"

      - name: Check logs
        if: always()
        run: |
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          cd ${GITHUB_WORKSPACE}/shakenfist
          tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh

      - name: Restart Shaken Fist nodes so we can collect logs
        if: always()
        run: |
          set -x
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          for node in ${sf1_uuid} ${sf2_uuid} ${sf3_uuid} ${sf4_uuid} ${sf5_uuid}; do
              sf-client instance reboot --hard ${node}
          done
          sleep 90

      - name: Fetch and tweak inventory
        if: always()
        run: |
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              debian@$primary:/etc/sf/inventory.yaml /srv/github/
          sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml

          echo "====="
          cat /srv/github/inventory.yaml

      - name: Gather logs
        if: always()
        run: |
          set -x

          # We need the ssh key in the place ansible expects it to be, which isn't
          # true on the CI worker node.
          cp /srv/github/id_ci /home/debian/.ssh/id_rsa
          cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub
          chown -R debian.debian /home/debian/.ssh

          ansible-playbook -i /srv/github/inventory.yaml \
              --extra-vars "base_image_user=debian ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \
              ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml

      - uses: actions/upload-artifact@v4
        if: always()
        with:
          name: bundle-node-lifecycle.zip
          retention-days: 90
          if-no-files-found: error
          path: /srv/github/artifacts/bundle.zip


  upgrade-0-6-15:
    needs: requirements-check
    runs-on: self-hosted
    concurrency:
      group: ${{ github.workflow }}-${{ github.ref }}-upgrade-0-6-15
      cancel-in-progress: true

    steps:
      - name: Set environment variables
        run: |
          echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
          echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV
          echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV

      - name: Checkout shakenfist
        uses: actions/checkout@v4
        with:
          path: shakenfist
          fetch-depth: 0

      - name: Determine if there is any dependency between the repositories
        run: |
          python3 ${GITHUB_WORKSPACE}/shakenfist/tools/clone_with_depends.py

      - name: Build infrastructure
        run: |
          cd ${GITHUB_WORKSPACE}/shakenfist
          ansible-playbook -i /home/debian/ansible-hosts \
              --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \
              base_image=sf://label/ci-images/debian-11 base_image_user=debian" \
              deploy/ansible/ci-topology-localhost-upgrade.yml

      - name: Copy CI tools to primary
        run: |
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          cd ${GITHUB_WORKSPACE}/shakenfist
          scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null -rp tools \
              debian@$primary:.

      - name: Run getsf to only install etcd, and then restore a backup
        run: |
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              debian@$primary /tmp/getsf-wrapper --tags bootstrap,etcd

          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              debian@$primary 'sudo chmod ugo+r /etc/sf/*; . /etc/sf/sfrc; /srv/shakenfist/venv/bin/sf-backup restore /srv/ci/backup-0.6.15-20230319'

      - name: Run getsf installer on primary
        run: |
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              debian@$primary /tmp/getsf-wrapper
          echo ""
          echo ""
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              debian@$primary \
              'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful'
          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
              -o UserKnownHostsFile=/dev/null \
              debian@$primary 'sudo chmod ugo+r /etc/sf/*'

      - name: Wait for five minutes
        run: sleep 300

      - name: Check logs
        if: always()
        run: |
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          cd ${GITHUB_WORKSPACE}/shakenfist
          tools/run_remote ${primary} sudo /home/${baseuser}/tools/ci_event_checks.sh

      - name: Fetch and tweak inventory
        if: always()
        run: |
          . ${GITHUB_WORKSPACE}/ci-environment.sh
          scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
              debian@$primary:/etc/sf/inventory.yaml /srv/github/
          sed -i 's|/root/.ssh|/home/debian/.ssh|g' /srv/github/inventory.yaml

          echo "====="
          cat /srv/github/inventory.yaml

      - name: Gather logs
        if: always()
        run: |
          set -x

          # We need the ssh key in the place ansible expects it to be, which isn't
          # true on the CI worker node.
          cp /srv/github/id_ci /home/debian/.ssh/id_rsa
          cp /srv/github/id_ci.pub /home/debian/.ssh/id_rsa.pub

          ansible-playbook -i /srv/github/inventory.yaml \
              --extra-vars "base_image_user=debian ansible_ssh_common_args='-o StrictHostKeyChecking=no'" \
              ${GITHUB_WORKSPACE}/shakenfist/deploy/ansible/ci-gather-logs.yml

      - uses: actions/upload-artifact@v4
        if: always()
        with:
          name: bundle-upgrade-0-6-15.zip
          retention-days: 90
          if-no-files-found: error
          path: /srv/github/artifacts/bundle.zip