Improve build group definitions (#788) #80
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Upload CI-tested images to Leafcloud S3 and sync clouds | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - 'environments/.stackhpc/tofu/cluster_image.auto.tfvars.json' | |
| env: | |
| S3_BUCKET: openhpc-images-prerelease | |
| IMAGE_PATH: environments/.stackhpc/tofu/cluster_image.auto.tfvars.json | |
| RUNNER_IMAGE_DIR: /mnt/images | |
| permissions: | |
| contents: read | |
| packages: write | |
| # To report GitHub Actions status checks | |
| statuses: write | |
| jobs: | |
| s3_cleanup: | |
| runs-on: ubuntu-22.04 | |
| concurrency: ${{ github.workflow }}-${{ github.ref }} | |
| strategy: | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Write s3cmd configuration | |
| run: | | |
| echo "${{ secrets['LEAFCLOUD_S3_CFG'] }}" > ~/.s3cfg | |
| shell: bash | |
| - name: Install s3cmd | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get --yes install s3cmd | |
| - name: Cleanup S3 bucket | |
| run: | | |
| s3cmd rm s3://${{ env.S3_BUCKET }} --recursive --force | |
| image_upload: | |
| runs-on: ubuntu-22.04 | |
| needs: s3_cleanup | |
| concurrency: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.build }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| build: | |
| - RL8 | |
| - RL9 | |
| env: | |
| ANSIBLE_FORCE_COLOR: True | |
| OS_CLOUD: openstack | |
| CI_CLOUD: ${{ vars.CI_CLOUD }} | |
| outputs: | |
| ci_cloud: ${{ steps.ci.outputs.CI_CLOUD }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Record which cloud CI is running on | |
| id: ci | |
| run: | | |
| echo "CI_CLOUD=${{ env.CI_CLOUD }}" >> "$GITHUB_OUTPUT" | |
| - name: Setup environment | |
| run: | | |
| python3 -m venv venv | |
| . venv/bin/activate | |
| pip install -U pip | |
| pip install "$(grep -o 'python-openstackclient[><=0-9\.]*' requirements.txt)" | |
| shell: bash | |
| - name: Write clouds.yaml | |
| run: | | |
| mkdir -p ~/.config/openstack/ | |
| echo "${{ secrets[format('{0}_CLOUDS_YAML', env.CI_CLOUD)] }}" > ~/.config/openstack/clouds.yaml | |
| shell: bash | |
| - name: Write s3cmd configuration | |
| run: | | |
| echo "${{ secrets['LEAFCLOUD_S3_CFG'] }}" > ~/.s3cfg | |
| shell: bash | |
| - name: Install s3cmd and qemu-utils | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get --yes install s3cmd qemu-utils | |
| - name: Retrieve image name | |
| run: | | |
| TARGET_IMAGE=$(jq --arg version "${{ matrix.build }}" -r '.cluster_image[$version]' "${{ env.IMAGE_PATH }}") | |
| echo "TARGET_IMAGE=${TARGET_IMAGE}" >> "$GITHUB_ENV" | |
| shell: bash | |
| - name: Download image to runner | |
| run: | | |
| . venv/bin/activate | |
| df -h | |
| sudo mkdir ${{ env.RUNNER_IMAGE_DIR }} | |
| sudo chmod ugo=rwX ${{ env.RUNNER_IMAGE_DIR }} | |
| openstack image save --file "${{ env.RUNNER_IMAGE_DIR }}/${{ env.TARGET_IMAGE }}.raw" "${{ env.TARGET_IMAGE }}" | |
| df -h | |
| shell: bash | |
| - name: Convert image to QCOW2 | |
| run: | | |
| . venv/bin/activate | |
| qemu-img convert -f raw -O qcow2 -c "${{ env.RUNNER_IMAGE_DIR }}/${{ env.TARGET_IMAGE }}.raw" "${{ env.RUNNER_IMAGE_DIR }}/${{ env.TARGET_IMAGE }}" | |
| shell: bash | |
| - name: Upload Image to S3 | |
| run: | | |
| echo "Uploading Image: ${{ env.TARGET_IMAGE }} to S3..." | |
| s3cmd --multipart-chunk-size-mb=150 put "${{ env.RUNNER_IMAGE_DIR }}/${{ env.TARGET_IMAGE }}" s3://${{ env.S3_BUCKET }} | |
| shell: bash | |
| image_sync: | |
| needs: image_upload | |
| runs-on: ubuntu-22.04 | |
| concurrency: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.cloud }}-${{ matrix.build }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| cloud: | |
| - LEAFCLOUD | |
| - SMS | |
| - ARCUS | |
| build: | |
| - RL8 | |
| - RL9 | |
| exclude: | |
| - cloud: ${{ needs.image_upload.outputs.ci_cloud }} | |
| env: | |
| ANSIBLE_FORCE_COLOR: True | |
| OS_CLOUD: openstack | |
| CI_CLOUD: ${{ matrix.cloud }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Record which cloud CI is running on | |
| run: | | |
| echo CI_CLOUD: ${{ env.CI_CLOUD }} | |
| - name: Setup environment | |
| run: | | |
| python3 -m venv venv | |
| . venv/bin/activate | |
| pip install -U pip | |
| pip install "$(grep -o 'python-openstackclient[><=0-9\.]*' requirements.txt)" | |
| shell: bash | |
| - name: Write clouds.yaml | |
| run: | | |
| mkdir -p ~/.config/openstack/ | |
| echo "${{ secrets[format('{0}_CLOUDS_YAML', env.CI_CLOUD)] }}" > ~/.config/openstack/clouds.yaml | |
| shell: bash | |
| - name: Retrieve image name | |
| run: | | |
| TARGET_IMAGE=$(jq --arg version "${{ matrix.build }}" -r '.cluster_image[$version]' "${{ env.IMAGE_PATH }}") | |
| echo "TARGET_IMAGE=${TARGET_IMAGE}" >> "$GITHUB_ENV" | |
| - name: Download latest image if missing | |
| run: | | |
| . venv/bin/activate | |
| bash .github/bin/get-s3-image.sh ${{ env.TARGET_IMAGE }} ${{ env.S3_BUCKET }} | |
| - name: Cleanup OpenStack Image (on error or cancellation) | |
| if: cancelled() || failure() | |
| run: | | |
| . venv/bin/activate | |
| image_hanging=$(openstack image list --name ${{ env.TARGET_IMAGE }} -f value -c ID -c Status | grep -v ' active$' | awk '{print $1}') | |
| if [ -n "$image_hanging" ]; then | |
| echo "Cleaning up OpenStack image with ID: $image_hanging" | |
| openstack image delete "$image_hanging" | |
| else | |
| echo "No image ID found, skipping cleanup." | |
| fi | |
| shell: bash |