[ISV-5336] Automation for pipeline re-run (#742)

Signed-off-by: Maurizio Porrato <[email protected]>

Author: mporrato

.github/workflows/build-and-test.yml

@@ -36,7 +36,7 @@ jobs:
       - name: Install Python dependencies
         run: |
           pdm sync -dG tox
-          python3 -m pip install ansible-lint
+          pipx install ansible-lint
 
       - name: Run Tests
         run: |
@@ -103,8 +103,8 @@ jobs:
 
       - name: Install dependencies
         run: |
-          # Install python dependencies
-          pip install --user openshift pygithub
+          # Install python packages needed from ansible
+          pipx inject ansible-core jmespath openshift pygithub
 
           # Add certificates to trusted list
           sudo cp  operator-pipeline-images/certs/* /usr/local/share/ca-certificates
@@ -146,5 +146,6 @@ jobs:
             -e "integration_tests_operator_bundle_version=0.2.${{ github.run_number }}-${{ github.run_attempt }}"
             -e "operator_pipeline_image_tag=${{ github.sha }}"
             -e "suffix=${{ steps.prepare.outputs.suffix }}"
+            -e "ansible_python_interpreter=/opt/pipx/venvs/ansible-core/bin/python3"
             --skip-tags=signing-pipeline
             -v

.github/workflows/deploy.yml

@@ -34,7 +34,8 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          pip install --user openshift
+          # Install python packages needed from ansible
+          pipx inject ansible-core jmespath openshift pygithub
       - name: Deploy dev environment
         uses: dawidd6/action-ansible-playbook@v2
         with:
@@ -43,7 +44,10 @@ jobs:
           requirements: playbooks/requirements.yml
           vault_password: ${{secrets.VAULT_PASSWORD}}
           options: |
-            --extra-vars "operator_pipeline_image_tag=${{ github.sha }} suffix=${{needs.prepare-env.outputs.short_sha}} env=dev"
+            --extra-vars "operator_pipeline_image_tag=${{ github.sha }}"
+            --extra-vars "suffix=${{needs.prepare-env.outputs.short_sha}}"
+            --extra-vars "env=dev"
+            --extra-vars "ansible_python_interpreter=/opt/pipx/venvs/ansible-core/bin/python3"
             --skip-tags ci,import-index-images
             --verbose
 
@@ -60,7 +64,8 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          pip install --user openshift
+          # Install python packages needed from ansible
+          pipx inject ansible-core jmespath openshift pygithub
       - name: Deploy qa environment
         uses: dawidd6/action-ansible-playbook@v2
         with:
@@ -69,7 +74,10 @@ jobs:
           requirements: playbooks/requirements.yml
           vault_password: ${{secrets.VAULT_PASSWORD}}
           options: |
-            --extra-vars "operator_pipeline_image_tag=${{ github.sha }} suffix=${{needs.prepare-env.outputs.short_sha}} env=qa"
+            --extra-vars "operator_pipeline_image_tag=${{ github.sha }}"
+            --extra-vars "suffix=${{needs.prepare-env.outputs.short_sha}}"
+            --extra-vars "env=qa"
+            --extra-vars "ansible_python_interpreter=/opt/pipx/venvs/ansible-core/bin/python3"
             --skip-tags ci,import-index-images
             --verbose
 
@@ -87,7 +95,8 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          pip install --user openshift
+          # Install python packages needed from ansible
+          pipx inject ansible-core jmespath openshift pygithub
       - name: Deploy stage environment
         uses: dawidd6/action-ansible-playbook@v2
         with:
@@ -96,7 +105,10 @@ jobs:
           requirements: playbooks/requirements.yml
           vault_password: ${{secrets.VAULT_PASSWORD}}
           options: |
-            --extra-vars "operator_pipeline_image_tag=${{ github.sha }} suffix=${{needs.prepare-env.outputs.short_sha}} env=stage"
+            --extra-vars "operator_pipeline_image_tag=${{ github.sha }}"
+            --extra-vars "suffix=${{needs.prepare-env.outputs.short_sha}}"
+            --extra-vars "env=stage"
+            --extra-vars "ansible_python_interpreter=/opt/pipx/venvs/ansible-core/bin/python3"
             --skip-tags ci,import-index-images
             --verbose
 
@@ -113,7 +125,8 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
-          pip install --user openshift
+          # Install python packages needed from ansible
+          pipx inject ansible-core jmespath openshift pygithub
       - name: Deploy prod environment
         uses: dawidd6/action-ansible-playbook@v2
         with:
@@ -122,7 +135,10 @@ jobs:
           requirements: playbooks/requirements.yml
           vault_password: ${{secrets.VAULT_PASSWORD_PROD}}
           options: |
-            --extra-vars "operator_pipeline_image_tag=${{ github.sha }} suffix=${{needs.prepare-env.outputs.short_sha}} env=prod"
+            --extra-vars "operator_pipeline_image_tag=${{ github.sha }}"
+            --extra-vars "suffix=${{needs.prepare-env.outputs.short_sha}}"
+            --extra-vars "env=prod"
+            --extra-vars "ansible_python_interpreter=/opt/pipx/venvs/ansible-core/bin/python3"
             --skip-tags ci,import-index-images
             --verbose
 

.github/workflows/documentation.yaml

@@ -15,7 +15,8 @@ jobs:
       - uses: actions/checkout@v4
       - name: Generate docs
         run: |
-          pip install mkdocs \
+          pipx install mkdocs
+          pipx inject mkdocs \
             mkdocs-material \
             pymdown-extensions \
             mkdocs-mermaid2-plugin

ansible/playbooks/requirements.yml

@@ -3,4 +3,4 @@ collections:
   - name: kubernetes.core
     version: 2.4.0
   - name: community.general
-    version: 7.5.0
+    version: 9.5.0

ansible/roles/integration_tests/files/wait_for_pipelinerun.sh

@@ -22,17 +22,16 @@
         var: pipeline_run.resources[0].metadata.name
 
     - name: "Wait for the run of {{ pipeline_name }}"
-      ansible.builtin.shell: |
-        ./wait_for_pipelinerun.sh \
-          {{ pipeline_run.resources[0].metadata.name }} \
-          {{ oc_namespace }}
-      args:
-        executable: /bin/bash
-        chdir: "{{ temp_tools_dir.path }}"
+      kubernetes.core.k8s_info:
+        api_version: tekton.dev/v1
+        kind: PipelineRun
+        namespace: "{{ oc_namespace }}"
+        name: "{{ pipeline_run.resources[0].metadata.name }}"
       register: pipeline_run_wait
-      failed_when: false
-      timeout: 5400  # 90 minutes
-      changed_when: false
+      until: >
+        pipeline_run_wait.resources[0] | community.general.json_query("contains(['False', 'True'], status.conditions[?type == 'Succeeded']|[-1].status)")
+      retries: 90
+      delay: 60
 
     - name: "Get pipelinerun logs - {{ pipeline_name }}"
       ansible.builtin.shell: |

ansible/roles/integration_tests/tasks/check_pipeline_run.yml

@@ -23,12 +23,3 @@
         remote_src: true
         include:
           - opm
-
-    - name: Copy a local scripts to the host
-      ansible.builtin.copy:
-        src: "{{ item }}"
-        dest: "{{ temp_tools_dir.path }}"
-        mode: a+x
-
-      with_items:
-        - "files/wait_for_pipelinerun.sh"

ansible/roles/integration_tests/tasks/tools.yml

@@ -67,6 +67,8 @@ Based on which pipeline fails one of these command can be used to re-trigger it
 
 After a pipeline is re-triggered using the command a few labels will be added and removed from the PR. First a new labels `pipeline/trigger-hosted` or `pipeline/trigger-release` is added. This label kick in the pipeline and pipeline itself start adding a labels based on the pipeline status.
 
+A script called `bulk-retrigger` is provided in the operator-pipeline container image to help re-triggering a pipeline on multiple PRs: it takes the repository name, a CSV file containing a list of PRs to process and automates the re-triggering of the pipeline one PR at a time. See the help text for details on how to run it.
+
 ### Pipeline logs
 Pipelines interacts with user using a Github Pull request interface. There are a slight differences between ISV and community repositories, but overall concept is the same.