Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Validate Platform | ||
|
Check failure on line 1 in .github/workflows/integration-build-platform.yml
|
||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| product_name: | ||
| required: true | ||
| type: string | ||
| ros_distro: | ||
| required: true | ||
| type: string | ||
| backend_name: | ||
| required: false | ||
| type: string | ||
| default: '' | ||
| debug_fleet_keep_alive: | ||
| required: false | ||
| type: boolean | ||
| fleet_ips: | ||
| required: true | ||
| type: string | ||
| fleet_number_members: | ||
| required: true | ||
| type: number | ||
| cluster: | ||
| required: false | ||
| default: "mary" | ||
| type: string | ||
| debug_simulation_tests_keep_alive: | ||
| required: false | ||
| type: boolean | ||
| default: false | ||
| proxmox_cluster_name: | ||
| required: false | ||
| type: string | ||
| default: hel0 | ||
| slack_channel: | ||
| required: false | ||
| type: string | ||
| default: "C02U028NMB7" | ||
| #default: "C02U028NMB7" # rnd-platform | ||
| # development slack channel | ||
| #default: "C05K2KF1UP8" | ||
| simulation_qa_key: | ||
| required: false | ||
| type: string | ||
| default: "simulator_tests" | ||
| simulation_qa_version: | ||
| required: false | ||
| type: string | ||
| default: "" | ||
| description: Version of the simulator tests to run. If not provided, the version will be taken from the product manifest. | ||
| with_fleet_tests: | ||
| required: false | ||
| type: string | ||
| default: false | ||
| description: Flag if we want the fleet job to run platform UI Fleet tests | ||
| fleet_tests_qa_key: | ||
| required: false | ||
| type: string | ||
| default: "ui_tests" | ||
| secrets: | ||
| auto_commit_user: | ||
| required: true | ||
| auto_commit_mail: | ||
| required: true | ||
| auto_commit_pwd: | ||
| required: true | ||
| registry_user: | ||
| required: true | ||
| registry_password: | ||
| required: true | ||
| nexus_publisher_user: | ||
| required: true | ||
| nexus_publisher_password: | ||
| required: true | ||
| gh_token: | ||
| required: true | ||
| aws_key_id: | ||
| required: true | ||
| aws_secret_key_id: | ||
| required: true | ||
| slack_token_id: | ||
| required: true | ||
| ssh_pem_fleet_aws_vm: | ||
| required: true | ||
| proxmox_ve_username: | ||
| required: true | ||
| proxmox_ve_password: | ||
| required: true | ||
| proxmox_ve_sim_username: | ||
| required: true | ||
| proxmox_ve_sim_password: | ||
| required: true | ||
| ssh_priv_key: | ||
| required: true | ||
| jira_username: | ||
| required: true | ||
| jira_password: | ||
| required: true | ||
| xray_clientid: | ||
| required: true | ||
| xray_secret: | ||
| required: true | ||
| minio_key_id: | ||
| required: true | ||
| minio_secret_key_id: | ||
| required: true | ||
| buildkite_token: | ||
| required: false | ||
| env: | ||
| CI_INTEGRATION_SCRIPTS_VERSION: "4.0.0.32" | ||
| MOBTEST_VERSION: "0.0.6.1" | ||
| PACKAGE_DEPLOYER_VERSION: "1.0.0.25" | ||
| GITHUB_API_USR: "OttoMation-Movai" | ||
| AWS_ACCESS_KEY_ID: ${{ secrets.aws_key_id }} | ||
| AWS_SECRET_ACCESS_KEY: ${{ secrets.aws_secret_key_id }} | ||
| AWS_DEFAULT_REGION: "us-east-1" | ||
| REGISTRY: registry.cloud.mov.ai | ||
| MID_REGISTRY: registry.hel.mov.ai | ||
| PUSH_REGISTRY: registry.cloud.mov.ai | ||
| ENV: qa | ||
| USERSPACE_FOLDER_PATH: /opt/movai/userspace | ||
| REMOTE_WORKSPACE_PATH: workspace | ||
| SIMULATION_ID: ci_simulation | ||
| XRAY_CLIENTID: ${{ secrets.xray_clientid}} | ||
| XRAY_SECRET: ${{ secrets.xray_secret}} | ||
| JIRA_USERNAME: ${{ secrets.jira_username}} | ||
| JIRA_PASSWORD: ${{ secrets.jira_password}} | ||
| PROVISIONING_ENV_VERSION: 0.1.0-17 | ||
| PROVISIONING_ENV_REPO: devops-tf-env-conf | ||
| PROVISION_INFRA_REPO: "devops-tf-proxmox-bpg" | ||
| PROVISION_INFRA_VERSION: "2.0.0-16" | ||
| ARTIFACTS_RETENTION_DAYS: 15 | ||
| jobs: | ||
| Validate-boostrap-configs: | ||
| runs-on: integration-pipeline | ||
| container: | ||
| image: registry.aws.cloud.mov.ai/qa/py-buildserver:v3.0.3 | ||
| credentials: | ||
| username: ${{secrets.registry_user}} | ||
| password: ${{secrets.registry_password}} | ||
| outputs: | ||
| slack_thread_id: ${{ steps.send-message.outputs.ts }} | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| - name: Validate Manifest | ||
| shell: bash | ||
| run: | | ||
| yamllint product-manifest.yaml | ||
| - name: Install CI Scripts in container | ||
| shell: bash | ||
| run: | | ||
| python3 -m pip install integration-pipeline==$CI_INTEGRATION_SCRIPTS_VERSION --ignore-installed | ||
| - name: raise version | ||
| run: | | ||
| git config --global --add safe.directory $(pwd) | ||
| git fetch | ||
| git checkout origin/${GITHUB_REF#refs/heads/} -- product.version | ||
| integration-pipeline raise | ||
| cat product.version | ||
| - name: Bootstraping simulator metadata | ||
| run: | | ||
| cat product.version | ||
| rm -rf simulator_artifacts ci_artifacts | ||
| integration-pipeline generate_meta_simulator_artifacts \ | ||
| --manifest_platform_base_key product_components \ | ||
| --product_name ${{ inputs.product_name }} \ | ||
| --branch ${GITHUB_REF#refs/heads/} | ||
| mkdir simulator_artifacts | ||
| cp ci_artifacts/* ./simulator_artifacts | ||
| - name: Bootstraping platform metadata | ||
| run: | | ||
| rm -rf ci_artifacts | ||
| integration-pipeline generate_meta_artifacts \ | ||
| --manifest_platform_base_key product_components \ | ||
| --override_backend ${{ inputs.backend_name }} | ||
| mkdir component_artifacts | ||
| cp ci_artifacts/* ./component_artifacts | ||
| - name: Stash component_artifacts | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: component_artifacts | ||
| path: component_artifacts/* | ||
| retention-days: ${{ env.ARTIFACTS_RETENTION_DAYS }} | ||
| - name: Stash robot_configs | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: robot_configs | ||
| path: "*.json*" | ||
| retention-days: ${{ env.ARTIFACTS_RETENTION_DAYS }} | ||
| - name: Stash sim_configs | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: sim_configs | ||
| path: simulator_artifacts/* | ||
| retention-days: ${{ env.ARTIFACTS_RETENTION_DAYS }} | ||
| - name: Pre-stash organization configs | ||
| run: | | ||
| rm -rf simulator_artifacts ci_artifacts platform_configs | ||
| mkdir platform_configs | ||
| cp product.version ./platform_configs/product.version | ||
| cp product-manifest.yaml ./platform_configs/product-manifest.yaml | ||
| - name: Prepare slack variables | ||
| if: always() | ||
| id: pre_slack | ||
| run: | | ||
| MESSAGE=":loading: CI: ${GITHUB_REPOSITORY} (${GITHUB_REF#refs/heads/}), build: $(cat product.version) (Attempt: #${{ github.run_attempt }}) is starting to be validated :mag_right: Details: https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" | ||
| echo "msg=${MESSAGE}" >> $GITHUB_OUTPUT | ||
| - name: Send Slack Message | ||
| uses: slackapi/slack-github-action@v2 | ||
| id: send-message | ||
| with: | ||
| method: chat.postMessage | ||
| payload: | | ||
| { | ||
| "channel": "${{ inputs.slack_channel }}", | ||
| "text": "${{ steps.pre_slack.outputs.msg }}" | ||
| } | ||
| token: ${{ secrets.slack_token_id }} | ||
| - name: Stash raised_meta | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: raised_meta | ||
| path: platform_configs/* | ||
| retention-days: ${{ env.ARTIFACTS_RETENTION_DAYS }} | ||
| Build-backend: | ||
| runs-on: integration-pipeline-${{ inputs.proxmox_cluster_name }} | ||
| needs: [Validate-boostrap-configs] | ||
| outputs: | ||
| slack_thread_id: ${{ needs.Validate-boostrap-configs.outputs.slack_thread_id }} | ||
| backend_image: ${{ steps.store_image_name.outputs.backend_image }} | ||
| steps: | ||
| - uses: rtCamp/action-cleanup@master | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| - name: Agent info | ||
| run: | | ||
| echo "public ip: $(curl ipinfo.io/ip)" | ||
| echo "private ip: $(hostname -I | awk '{print $1}')" | ||
| - name: Setup CI Scripts in .ci-venv | ||
| shell: bash | ||
| run: | | ||
| python3 -m venv .ci-venv --clear | ||
| . .ci-venv/bin/activate | ||
| [ -f ci-requirements.txt ] && pip install -r ci-requirements.txt | ||
| python3 -m pip install --upgrade pyOpenSSL cryptography | ||
| python3 -m pip install integration-pipeline==$CI_INTEGRATION_SCRIPTS_VERSION --ignore-installed | ||
| - name: unstash raised_meta | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| name: raised_meta | ||
| path: platform_configs | ||
| - name: unstash component_artifacts | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| name: component_artifacts | ||
| - name: extract product configs, manifest to main folder | ||
| shell: bash | ||
| run: | | ||
| cp ./platform_configs/product.version product.version | ||
| cp ./platform_configs/product-manifest.yaml product-manifest.yaml | ||
| - name: Lint docker image | ||
| shell: bash | ||
| run: | | ||
| docker run --rm -i -e HADOLINT_FAILURE_THRESHOLD=error hadolint/hadolint < docker/backend/Dockerfile | ||
| - name: Login to ${{ env.REGISTRY }} Registry | ||
| uses: docker/login-action@v3 | ||
| with: | ||
| username: ${{ secrets.registry_user }} | ||
| password: ${{ secrets.registry_password }} | ||
| registry: ${{ env.REGISTRY }} | ||
| - name: Prepare docker build variables | ||
| id: pre_build | ||
| shell: bash | ||
| run: | | ||
| echo "base_name=$(cat backend_base_name.ci)" >> $GITHUB_OUTPUT | ||
| echo "base_version=$(cat backend_base_version.ci)" >> $GITHUB_OUTPUT | ||
| echo "raised_version=$(cat product.version)" >> $GITHUB_OUTPUT | ||
| - name: Set up Docker Buildx | ||
| uses: docker/setup-buildx-action@v3 | ||
| - name: Docker build | ||
| shell: bash | ||
| run: | | ||
| docker build --add-host ${{ env.REGISTRY }}:172.22.0.106 \ | ||
| --build-arg DOCKER_REGISTRY=${{ env.REGISTRY }} \ | ||
| --build-arg BASE_IMAGE=${{ steps.pre_build.outputs.base_name }} \ | ||
| --build-arg TAG=${{ steps.pre_build.outputs.base_version }} \ | ||
| --build-arg CI_SCRIPT_VERSION=${{ env.CI_INTEGRATION_SCRIPTS_VERSION }} \ | ||
| --file docker/backend/Dockerfile \ | ||
| --platform linux/amd64 \ | ||
| --tag ${{ env.REGISTRY }}/ci/${{ inputs.backend_name }}:${{ steps.pre_build.outputs.raised_version }} \ | ||
| --pull \ | ||
| --push . | ||
| - name: store image name | ||
| id: store_image_name | ||
| shell: bash | ||
| run: | | ||
| echo "backend_image=${{ inputs.backend_name }}:${{ steps.pre_build.outputs.raised_version }}" >> $GITHUB_OUTPUT | ||
| - name: Get current job id | ||
| if: always() | ||
| shell: bash | ||
| id: job_info | ||
| run: | | ||
| sudo apt update | ||
| sudo apt install jq -y | ||
| job_id=$(gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id}}/attempts/${{ github.run_attempt }}/jobs | jq -r '.jobs | .[0].id') | ||
| job_html_url=$(gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id}}/attempts/${{ github.run_attempt }}/jobs | jq -r '.jobs | map(select(.name | contains("${{ github.job }}"))) | .[0].html_url') | ||
| echo "job_url=$job_html_url" | tee >> $GITHUB_OUTPUT | ||
| env: | ||
| GITHUB_TOKEN: ${{ secrets.gh_token }} | ||
| - name: Prepare slack variables | ||
| if: always() | ||
| id: pre_slack_result | ||
| run: | | ||
| MESSAGE=":white_check_mark: ${{ github.job }} (Attempt: #${{ github.run_attempt }}) job passed" | ||
| MESSAGE_ERR=":x: ${{ github.job }} (Attempt: #${{ github.run_attempt }}) job failed" | ||
| echo "msg=${MESSAGE}" >> $GITHUB_OUTPUT | ||
| echo "msg_error=${MESSAGE_ERR}\n Details: ${{ steps.job_info.outputs.job_url }}" >> $GITHUB_OUTPUT | ||
| - name: Slack message success | ||
| uses: slackapi/slack-github-action@v2 | ||
| with: | ||
| method: chat.postMessage | ||
| payload: | | ||
| { | ||
| "channel": "${{ inputs.slack_channel }}", | ||
| "text": "${{ steps.pre_slack_result.outputs.msg }}", | ||
| "thread_ts": "${{ needs.Validate-boostrap-configs.outputs.slack_thread_id }}" | ||
| } | ||
| token: ${{ secrets.slack_token_id }} | ||
| - name: Slack message failure | ||
| uses: slackapi/slack-github-action@v2 | ||
| if: failure() | ||
| with: | ||
| method: chat.postMessage | ||
| payload: | | ||
| { | ||
| "channel": "${{ inputs.slack_channel }}", | ||
| "text": "${{ steps.pre_slack_result.outputs.msg_error }}", | ||
| "thread_ts": "${{ needs.Validate-boostrap-configs.outputs.slack_thread_id }}" | ||
| } | ||
| token: ${{ secrets.slack_token_id }} | ||
| - name: Docker cleanups | ||
| if: always() | ||
| shell: bash | ||
| run: | | ||
| docker system prune -f | ||
| docker image prune --all -f | ||
| Fleet-Validations: | ||
| needs: [Build-backend] | ||
| runs-on: ip-${{ inputs.proxmox_cluster_name }}-queuer | ||
| steps: | ||
| - name: Cleanup Workspace | ||
| uses: rtCamp/action-cleanup@master | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| - name: Agent info | ||
| id: agent_info | ||
| run: | | ||
| echo "ip=$(hostname -I | awk '{print $1}')" | tee $GITHUB_OUTPUT | ||
| - name: Setup CI Scripts in .ci-venv | ||
| shell: bash | ||
| run: | | ||
| python3 -m venv .ci-venv --clear | ||
| . .ci-venv/bin/activate | ||
| [ -f ci-requirements.txt ] && pip install -r ci-requirements.txt | ||
| python3 -m pip install --upgrade pyOpenSSL cryptography | ||
| python3 -m pip install integration-pipeline==$CI_INTEGRATION_SCRIPTS_VERSION --ignore-installed | ||
| - name: unstash robot_configs | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| name: robot_configs | ||
| path: . | ||
| - name: Setup infra environment configs | ||
| id: infra_env_configs_setup | ||
| shell: bash | ||
| run: | | ||
| env_configs_dir=infra_env_configs | ||
| env_configs_version=${{ env.PROVISIONING_ENV_VERSION }} | ||
| env_configs_repo_name=${{ env.PROVISIONING_ENV_REPO }} | ||
| rm -rf $env_configs_dir | ||
| . .ci-venv/bin/activate | ||
| integration-pipeline fetch_by_tag --repo $env_configs_repo_name --version $env_configs_version --gh_api_user $GITHUB_API_USR --gh_api_pwd ${{ secrets.auto_commit_pwd }} --target_dir $env_configs_dir | ||
| ls -la $env_configs_dir | ||
| echo "target_dir=${env_configs_dir}" >> $GITHUB_OUTPUT | ||
| deactivate | ||
| - name: Setup terraform proxmox provisioner | ||
| id: provision_infra_setup | ||
| shell: bash | ||
| run: | | ||
| provision_infra_dir=provision_scripts | ||
| provision_infra_version=${{ env .PROVISION_INFRA_VERSION}} | ||
| provision_infra_repo_name=${{ env.PROVISION_INFRA_REPO }} | ||
| rm -rf $provision_infra_dir | ||
| . .ci-venv/bin/activate | ||
| integration-pipeline fetch_by_tag --repo $provision_infra_repo_name --version $provision_infra_version --gh_api_user $GITHUB_API_USR --gh_api_pwd ${{ secrets.auto_commit_pwd }} --target_dir $provision_infra_dir | ||
| deactivate | ||
| ls -la $provision_infra_dir | ||
| echo "target_dir=${provision_infra_dir}" >> $GITHUB_OUTPUT | ||
| - name: Define Instance names | ||
| id: infra_names | ||
| shell: bash | ||
| run: | | ||
| branch=$(echo ${GITHUB_REF#refs/heads/} | sed "s;\.;-;g" ) | ||
| local_manager_prefix="ip-$branch-manager" | ||
| local_worker_prefix="ip-$branch-worker" | ||
| echo "$local_manager_prefix" | ||
| echo "$local_worker_prefix" | ||
| total_resources=${{ inputs.fleet_number_members }} | ||
| ((total_resources+=1)) | ||
| echo "manager_prefix=${local_manager_prefix}" >> $GITHUB_OUTPUT | ||
| echo "worker_prefix=${local_worker_prefix}" >> $GITHUB_OUTPUT | ||
| echo "total_resources=${total_resources}" >> $GITHUB_OUTPUT | ||
| - name: Provision remote vms (Proxmox) | ||
| working-directory: ${{ steps.provision_infra_setup.outputs.target_dir }} | ||
| shell: bash | ||
| run: | | ||
| echo "${{ secrets.ssh_pem_fleet_aws_vm }}" > ~/.ssh/aws_slave.pem | ||
| sudo chmod 600 ~/.ssh/aws_slave.pem | ||
| sudo apt install jq -y | ||
| multiply_node=$(printf '"${{ inputs.proxmox_cluster_name }}",%.0s' {1..${{ steps.infra_names.outputs.total_resources }}}) | ||
| node_list_str=${multiply_node::-1} | ||
| var_file_arg='-var-file=../${{ steps.infra_env_configs_setup.outputs.target_dir }}/hel/${{ inputs.proxmox_cluster_name }}_fleet_test.tfvars' | ||
| echo "proxmox_host_list=[$node_list_str]">>input.tfvars | ||
| echo "fleet_peer_nr=${{ inputs.fleet_number_members }}">>input.tfvars | ||
| echo 'fleet_password="n/a"'>>input.tfvars | ||
| echo 'fleet_manager_name="${{ steps.infra_names.outputs.manager_prefix }}"'>>input.tfvars | ||
| echo 'fleet_peer_name_prefix="${{ steps.infra_names.outputs.worker_prefix }}"'>>input.tfvars | ||
| echo 'ip_list=${{ inputs.fleet_ips }}'>>input.tfvars | ||
| echo 'proxmox_ve_username="${{ secrets.proxmox_ve_username }}"'>>input.tfvars | ||
| echo 'proxmox_ve_password="${{ secrets.proxmox_ve_password }}"'>>input.tfvars | ||
| echo "\n">>input | ||
| echo "File args: $var_file_arg" | ||
| echo "Input File args: $(cat input.tfvars)" | ||
| terraform init -backend-config="key=${{ inputs.proxmox_cluster_name }}-fleet-${{ steps.infra_names.outputs.manager_prefix }}.tfstate" | ||
| terraform apply -auto-approve $var_file_arg -var-file=input.tfvars | ||
| terraform refresh $var_file_arg -var-file=input.tfvars | ||
| - name: Check tf data fetch output | ||
| if: failure() | ||
| shell: bash | ||
| continue-on-error: true | ||
| run: cat /tmp/terraform-external-debug.log | ||
| - name: Gather Terraform outputs | ||
| working-directory: ${{ steps.provision_infra_setup.outputs.target_dir }} | ||
| id: infra_outputs | ||
| shell: bash | ||
| run: | | ||
| ip=$(terraform output manager_ip_address) | ||
| user=$(terraform output user) | ||
| members=$(terraform output peers_ip_addresses) | ||
| echo "host_ip=$(echo $ip | sed "s;\";;g")" >> $GITHUB_OUTPUT | ||
| echo "host_user=$(echo $user | sed "s;\";;g")" >> $GITHUB_OUTPUT | ||
| echo "ssh_connect_string=$(echo $user | sed "s;\";;g")@$(echo $ip | sed "s;\";;g")" >> $GITHUB_OUTPUT | ||
| echo "fleet_ips=$(echo $members | sed "s;\";;g")" >> $GITHUB_OUTPUT | ||
| - name: Prepare Devops provisioning slack message | ||
| if: always() | ||
| id: pre_slack_infra | ||
| run: | | ||
| MESSAGE_ERR=":x: CI: ${GITHUB_REPOSITORY}, (${GITHUB_REF#refs/heads/}), build: $(cat product.version) is being impacted by an infrastructural issue. \ | ||
| Provisioning of fleet infrastructure failed. Please take a look! \ | ||
| Details: https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" | ||
| echo "msg_error=${MESSAGE_ERR}" >> $GITHUB_OUTPUT | ||
| - name: Slack message failure | ||
| uses: slackapi/slack-github-action@v2 | ||
| if: failure() | ||
| with: | ||
| method: chat.postMessage | ||
| payload: | | ||
| { | ||
| "channel": "G0102LEV1CL", | ||
| "text": "${{ steps.pre_slack_infra.outputs.msg_error }}" | ||
| } | ||
| token: ${{ secrets.slack_token_id }} | ||
| - name: Apply ansible inventory | ||
| shell: bash | ||
| run: | | ||
| . .ci-venv/bin/activate | ||
| cp ${{ steps.provision_infra_setup.outputs.target_dir }}/provisioned_inventory.yml staging/provisioned_inventory.yml | ||
| cat staging/provisioned_inventory.yml | ||
| integration-pipeline get_yml_value --file staging/provisioned_inventory.yml --key fleet.children.managers.hosts.manager.ansible_host --output_file ./staging/manager_private_ip.txt | ||
| deactivate | ||
| - name: Setup ansible installation | ||
| id: ansible_install_setup | ||
| shell: bash | ||
| env: | ||
| install_key: ansible_deploy | ||
| run: | | ||
| rm -f /tmp/target_dir.txt /tmp/version.txt /tmp/repo_name.txt | ||
| . .ci-venv/bin/activate | ||
| integration-pipeline get_yml_value --file product-manifest.yaml --key product_components.installion.$install_key.target_dir --output_file /tmp/target_dir.txt | ||
| integration-pipeline get_yml_value --file product-manifest.yaml --key product_components.installion.$install_key.version --output_file /tmp/version.txt | ||
| integration-pipeline get_yml_value --file product-manifest.yaml --key product_components.installion.$install_key.name --output_file /tmp/repo_name.txt | ||
| install_infra_dir=$(cat /tmp/target_dir.txt) | ||
| install_infra_version=$(cat /tmp/version.txt) | ||
| install_infra_repo_name=$(cat /tmp/repo_name.txt) | ||
| rm -rf $install_infra_repo_name | ||
| integration-pipeline fetch_by_tag --repo $install_infra_repo_name --version $install_infra_version --gh_api_user $GITHUB_API_USR --gh_api_pwd ${{ secrets.auto_commit_pwd }} --target_dir $install_infra_dir | ||
| ls -la $install_infra_dir | ||
| echo "target_dir=${install_infra_dir}" >> $GITHUB_OUTPUT | ||
| deactivate | ||
| - name: Login to ${{ env.REGISTRY }} Registry | ||
| uses: docker/login-action@v3 | ||
| with: | ||
| username: ${{ secrets.registry_user }} | ||
| password: ${{ secrets.registry_password }} | ||
| registry: ${{ env.REGISTRY }} | ||
| - name: Ansible install platform | ||
| if: failure() | ||
| id: ansible_install_platform | ||
| working-directory: ${{ steps.ansible_install_setup.outputs.target_dir }} | ||
| shell: bash | ||
| run: | | ||
| echo "${{ secrets.ssh_pem_fleet_aws_vm }}" > ~/.ssh/aws_slave.pem | ||
| sudo chmod 600 ~/.ssh/aws_slave.pem | ||
| while sudo fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1 ; do echo Waiting for other software managers to finish... ; sleep 5;done | ||
| python3.9 -m venv ansible-venv | ||
| source ansible-venv/bin/activate | ||
| python3 -m pip install -r requirements.txt | ||
| ansible-galaxy install -r requirements.yml --timeout 120 | ||
| touch ~/.ssh/known_hosts | ||
| sudo chmod 600 ~/.ssh/known_hosts | ||
| echo "Peer ips:${{ steps.infra_outputs.outputs.fleet_ips }}" | ||
| # Ensure cloud init is done on all the hosts | ||
| members=("manager") | ||
| for i in $(seq 0 $(( ${{ inputs.fleet_number_members}} - 1 ))); do | ||
| members+=("member$i") | ||
| done | ||
| REGISTRY=${{ env.REGISTRY }} | ||
| BACKEND_IMAGE=${{ needs.Build-backend.outputs.backend_image }} | ||
| MID_REGISTRY=${{ env.MID_REGISTRY }} | ||
| REGISTRY_USER=${{ secrets.registry_user }} | ||
| REGISTRY_PASS=${{ secrets.registry_password }} | ||
| SOURCE_TAG="$REGISTRY/ci/$BACKEND_IMAGE" | ||
| TARGET_TAG_REGISTRY="$REGISTRY/qa/$BACKEND_IMAGE" | ||
| TARGET_TAG_MID_REGISTRY="$MID_REGISTRY/qa/$BACKEND_IMAGE" | ||
| for fleet_host in ${members[@]}; do | ||
| ansible $fleet_host -i ../staging/provisioned_inventory.yml --key-file ~/.ssh/aws_slave.pem -m shell -a "set -e; docker login $REGISTRY -u $REGISTRY_USER -p $REGISTRY_PASS" | ||
| ansible $fleet_host -i ../staging/provisioned_inventory.yml --key-file ~/.ssh/aws_slave.pem -m shell -a "set -e; docker pull $SOURCE_TAG" | ||
| ansible $fleet_host -i ../staging/provisioned_inventory.yml --key-file ~/.ssh/aws_slave.pem -m shell -a "set -e; docker tag $SOURCE_TAG $TARGET_TAG_REGISTRY" | ||
| ansible $fleet_host -i ../staging/provisioned_inventory.yml --key-file ~/.ssh/aws_slave.pem -m shell -a "set -e; docker tag $SOURCE_TAG $TARGET_TAG_MID_REGISTRY" | ||
| done | ||
| ansible-playbook install.yml \ | ||
| -i ../staging/provisioned_inventory.yml \ | ||
| --key-file ~/.ssh/aws_slave.pem \ | ||
| --extra-vars=@"$(pwd)/.."/product-manifest.yaml \ | ||
| -e fleet_domain_dns="" \ | ||
| -e "{\"proxycerts__remote_redis_servers_fqn\": [$(cat ../staging/manager_private_ip.txt)]}" \ | ||
| -e "{\"fleet_manager_san\": [ \"DNS:$(cat ../staging/manager_private_ip.txt)\" ]}" \ | ||
| -e "{\"fleet_extra_hosts\": [ \"10.26.0.10 registry.${{ inputs.proxmox_cluster_name }}.mov.ai traefik\", \"$(cat ../staging/manager_private_ip.txt) manager ${{ steps.infra_names.outputs.manager_prefix }}\" ]}" \ | ||
| --skip-tags "validate,ufw,hardening" | ||
| execution_status=$? | ||
| deactivate | ||
| exit $execution_status | ||
| - name: Configure SSH | ||
| if: ${{ inputs.with_fleet_tests }} | ||
| id: remote_ssh_setup | ||
| shell: bash | ||
| run: | | ||
| ssh-keygen -f ~/.ssh/known_hosts -R ${{ steps.infra_outputs.outputs.host_ip }} || true | ||
| ssh-keyscan -H ${{ steps.infra_outputs.outputs.host_ip }} >> ~/.ssh/known_hosts || true | ||
| ssh ${{ steps.infra_outputs.outputs.ssh_connect_string }} -i ~/.ssh/aws_slave.pem -o StrictHostKeyChecking=no ' | ||
| set -e | ||
| cloud-init status --wait; rm -rf ./${{ env.REMOTE_WORKSPACE_PATH }}; mkdir -p ./${{ env.REMOTE_WORKSPACE_PATH }} | ||
| - name: Remote - Agent info | ||
| if: ${{ inputs.with_fleet_tests }} | ||
| id: agent_info | ||
| run: | | ||
| ssh ${{ steps.infra_outputs.outputs.ssh_connect_string }} -i ~/.ssh/aws_slave.pem -o StrictHostKeyChecking=no ' | ||
| set -e | ||
| ip=$(hostname -I | cut -d " " -f1) | ||
| echo $ip | ||
| ' | ||
| - name: Remote - Install CI Scripts | ||
| if: ${{ inputs.with_fleet_tests }} | ||
| shell: bash | ||
| run: ssh ${{ steps.infra_outputs.outputs.ssh_connect_string }} -i ~/.ssh/aws_slave.pem -o StrictHostKeyChecking=no "set -e ; python3 -m pip install integration-pipeline==$CI_INTEGRATION_SCRIPTS_VERSION" | ||
| - name: Remote - unstash manifest | ||
| if: ${{ inputs.with_fleet_tests }} | ||
| shell: bash | ||
| run: scp -i ~/.ssh/aws_slave.pem -o StrictHostKeyChecking=no product-manifest.yaml ${{ steps.infra_outputs.outputs.ssh_connect_string }}:${{ env.REMOTE_WORKSPACE_PATH }}/product-manifest.yaml | ||
| - name: Remote - unstash robot_jsons_noetic | ||
| if: ${{ inputs.with_fleet_tests }} | ||
| shell: bash | ||
| run: scp -r -i ~/.ssh/aws_slave.pem -o StrictHostKeyChecking=no *.json* ${{ steps.infra_outputs.outputs.ssh_connect_string }}:${{ env.REMOTE_WORKSPACE_PATH }}/ | ||
| - name: Remote - Setup QA ${{ inputs.fleet_tests_qa_key}} tests | ||
| if: ${{ inputs.with_fleet_tests }} | ||
| shell: bash | ||
| id: simulator_tests_setup | ||
| run: | | ||
| ssh ${{ steps.infra_outputs.outputs.ssh_connect_string }} -i ~/.ssh/aws_slave.pem -o StrictHostKeyChecking=no ' | ||
| set -e | ||
| cd ${{ env.REMOTE_WORKSPACE_PATH }} | ||
| qa_key=${{ inputs.fleet_tests_qa_key}} | ||
| rm -f /tmp/target_dir.txt /tmp/version.txt /tmp/repo_name.txt /tmp/test_set.txt | ||
| export PATH="$HOME/.local/bin:$PATH" | ||
| integration-pipeline get_yml_value --file product-manifest.yaml --key product_components.qa.$qa_key.target_dir --output_file /tmp/target_dir.txt | ||
| integration-pipeline get_yml_value --file product-manifest.yaml --key product_components.qa.$qa_key.version --output_file /tmp/version.txt | ||
| integration-pipeline get_yml_value --file product-manifest.yaml --key product_components.qa.$qa_key.name --output_file /tmp/repo_name.txt | ||
| integration-pipeline get_yml_value --file product-manifest.yaml --key product_components.qa.$qa_key.test_set --output_file /tmp/test_set.txt | ||
| integration-pipeline get_yml_value --file product-manifest.yaml --key product_components.qa.$qa_key.test_dir --output_file /tmp/test_dir.txt | true | ||
| tests_dir=$(cat /tmp/target_dir.txt) | ||
| tests_version=$(cat /tmp/version.txt) | ||
| tests_repo_name=$(cat /tmp/repo_name.txt) | ||
| test_set=$(cat /tmp/test_set.txt) | ||
| test_exec_dir="." | ||
| if [ -f /tmp/test_dir.txt ]; then | ||
| test_exec_dir="$test_exec_dir/$(cat /tmp/test_dir.txt)" | ||
| fi | ||
| echo "$test_exec_dir">/tmp/test_exec_dir.txt | ||
| rm -rf $tests_repo_name | ||
| integration-pipeline fetch_by_tag --repo $tests_repo_name --version $tests_version --gh_api_user ${{ env.GITHUB_API_USR }} --gh_api_pwd ${{ secrets.auto_commit_pwd }} --target_dir $tests_dir | ||
| ls -la $tests_dir | ||
| # install test dependencies on host | ||
| sudo apt install -y --allow-downgrades python3-venv python3-rosnode python3-rosparam python3-rostopic | ||
| # setup venv in a step that is always executed | ||
| pushd "${tests_dir}" | ||
| python3 -m venv venv --clear | ||
| . venv/bin/activate | ||
| pip install -i https://artifacts.cloud.mov.ai/repository/pypi-integration/simple --extra-index-url https://pypi.org/simple -r requirements.txt | ||
| deactivate | ||
| popd | ||
| ' | ||
| rm -f /tmp/target_dir.txt /tmp/version.txt /tmp/repo_name.txt /tmp/test_set.txt /tmp/test_exec_dir.txt | ||
| scp -r -i ~/.ssh/aws_slave.pem -o StrictHostKeyChecking=no ${{ steps.infra_outputs.outputs.ssh_connect_string }}:/tmp/*.txt /tmp/ | ||
| echo "target_dir=$(cat /tmp/target_dir.txt)" >> $GITHUB_OUTPUT | ||
| echo "version=$(cat /tmp/version.txt)" >> $GITHUB_OUTPUT | ||
| echo "test_set=$(cat /tmp/test_set.txt)" >> $GITHUB_OUTPUT | ||
| echo "test_exec_dir=$(cat /tmp/test_exec_dir.txt)" >> $GITHUB_OUTPUT | ||
| - name: Remote - Run Fleet [${{ inputs.fleet_tests_qa_key}}] | ||
| if: ${{ inputs.with_fleet_tests }} | ||
| id: run_tests | ||
| shell: bash | ||
| run: | | ||
| ssh ${{ steps.infra_outputs.outputs.ssh_connect_string }} -i ~/.ssh/aws_slave.pem -o StrictHostKeyChecking=no ' | ||
| set -e | ||
| export selenoid_url="http://selenoid-ui.hel.mov.ai" | ||
| export host_ip=${{ steps.agent_info.outputs.ip }} | ||
| export movai_user=movai | ||
| export movai_pwd=movai123 | ||
| export test_set=${{ steps.infra_tests_setup.outputs.test_set }} | ||
| movai-cli user manager $movai_user $movai_pwd || true | ||
| . test-venv/bin/activate | ||
| echo "python3 -m pytest ${{ inputs.pytest_args}}" | ||
| python3 -m pytest tests/cucumber/test_fleet_board.py -ra --hub_url $selenoid_url --base_url https://$host_ip/ --movai-user $movai_user --movai-pw $movai_pwd --cucumberjson=./results.json --junit-xml=junit.xml | tee pytest-output.log | ||
| deactivate | ||
| ' | ||
| - name: Get current job id | ||
| if: always() | ||
| shell: bash | ||
| id: job_info | ||
| run: | | ||
| sudo apt update | ||
| sudo apt install jq -y | ||
| job_id=$(gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id}}/attempts/${{ github.run_attempt }}/jobs | jq -r '.jobs | .[0].id') | ||
| job_html_url=$(gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id}}/attempts/${{ github.run_attempt }}/jobs | jq -r '.jobs | map(select(.name | contains("${{ github.job }}"))) | .[0].html_url') | ||
| echo "$job_id" | ||
| echo "$job_html_url" | ||
| echo "job_url=$job_html_url" >> $GITHUB_OUTPUT | ||
| env: | ||
| GITHUB_TOKEN: ${{ secrets.gh_token }} | ||
| - name: Prepare slack variables | ||
| if: always() | ||
| id: pre_slack_result | ||
| run: | | ||
| MESSAGE=":white_check_mark: ${{ github.job }} (Attempt: #${{ github.run_attempt }}) job passed" | ||
| MESSAGE_ERR=":x: ${{ github.job }} (Attempt: #${{ github.run_attempt }}) job failed" | ||
| echo "msg=${MESSAGE}" >> $GITHUB_OUTPUT | ||
| echo "msg_error=${MESSAGE_ERR}\n Details: ${{ steps.job_info.outputs.job_url }}" >> $GITHUB_OUTPUT | ||
| - name: Slack message success | ||
| uses: slackapi/slack-github-action@v2 | ||
| with: | ||
| method: chat.postMessage | ||
| payload: | | ||
| { | ||
| "channel": "${{ inputs.slack_channel }}", | ||
| "text": "${{ steps.pre_slack_result.outputs.msg }}", | ||
| "thread_ts": "${{ needs.Build-backend.outputs.slack_thread_id }}" | ||
| } | ||
| token: ${{ secrets.slack_token_id }} | ||
| - name: Slack message failure | ||
| uses: slackapi/slack-github-action@v2 | ||
| if: failure() | ||
| with: | ||
| method: chat.postMessage | ||
| payload: | | ||
| { | ||
| "channel": "${{ inputs.slack_channel }}", | ||
| "text": "${{ steps.pre_slack_result.outputs.msg_error }}", | ||
| "thread_ts": "${{ needs.Build-backend.outputs.slack_thread_id }}" | ||
| } | ||
| token: ${{ secrets.slack_token_id }} | ||
| - name: Collect Fleet QA artifacts | ||
| working-directory: ${{ steps.ansible_install_setup.outputs.target_dir }} | ||
| if: always() | ||
| shell: bash | ||
| env: | ||
| API_DIR: ${{ steps.api_tests_setup.outputs.target_dir }} | ||
| run: | | ||
| rm -rf fleet_qa_artifacts | ||
| mkdir -p fleet_qa_artifacts/install | ||
| source ansible-venv/bin/activate | ||
| # install fleet_tests artifacts | ||
| for fleet_host in "manager" "member0" "member1"; do | ||
| ansible $fleet_host -i ../staging/provisioned_inventory.yml --key-file ~/.ssh/aws_slave.pem -m shell -a 'journalctl -u movai-service --since "1hour ago"' > fleet_qa_artifacts/install/$fleet_host.log || true | ||
| echo "From $fleet_host:" | ||
| ansible $fleet_host -i ../staging/provisioned_inventory.yml --key-file ~/.ssh/aws_slave.pem -m shell -a 'docker ps -a' > fleet_qa_artifacts/install/$fleet_host-docker_ps.log || true | ||
| tail -n +2 fleet_qa_artifacts/install/$fleet_host-docker_ps.log | ||
| ansible $fleet_host -i ../staging/provisioned_inventory.yml --key-file ~/.ssh/aws_slave.pem -m shell -a 'journalctl -u docker --boot --lines=all' > fleet_qa_artifacts/install/$fleet_host-all-docker.log || true | ||
| done | ||
| deactivate | ||
| # qa api tests artifacts | ||
| # *.log and *.zip might not exist if the test fails early | ||
| mkdir -p fleet_qa_artifacts/api | ||
| cp -r "${API_DIR}"/*.{log,tar} "${API_DIR}"/results/*.zip fleet_qa_artifacts/api 2>/dev/null || true | ||
| - name: Stash Fleet QA artifacts | ||
| if: always() | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: ${{ github.run_number }}_${{ github.run_attempt }}_fleet_qa_artifacts_${{ github.run_id }} | ||
| path: ${{ steps.ansible_install_setup.outputs.target_dir }}/fleet_qa_artifacts/* | ||
| retention-days: ${{ env.ARTIFACTS_RETENTION_DAYS }} | ||
| - name: Teardown remote vms (Proxmox) | ||
| working-directory: ${{ steps.provision_infra_setup.outputs.target_dir }} | ||
| if: ${{ ( !inputs.debug_fleet_keep_alive && success() ) || cancelled() || ( !inputs.debug_fleet_keep_alive && failure() ) }} | ||
| shell: bash | ||
| run: | | ||
| set +e | ||
| attempts=3 | ||
| count=0 | ||
| exit_status=-1 | ||
| while [ $count -lt $attempts ]; do | ||
| var_file_arg='-var-file=../${{ steps.infra_env_configs_setup.outputs.target_dir }}/hel/${{ inputs.proxmox_cluster_name }}_fleet_test.tfvars' | ||
| terraform destroy -auto-approve $var_file_arg -var-file=input.tfvars -var "fleet_peer_nr=0" -var "manager_ip_lookup=false" | ||
| exit_status=$? | ||
| if [ $exit_status -eq 0 ]; then | ||
| break | ||
| else | ||
| ((count++)) | ||
| echo "Retrying Terraform destroy (attempt $count)..." | ||
| fi | ||
| done | ||
| set -e | ||
| echo "Terraform destroy returned with exit status ${exit_status}. Attempted $count times." | ||
| exit ${exit_status} | ||
