ci: update workflows on release/25.10-lts from main (#180) #1190
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and test | ||
| on: | ||
| workflow_dispatch: | ||
| pull_request: | ||
| types: | ||
| - opened | ||
| - synchronize | ||
| - reopened | ||
| - labeled | ||
| push: | ||
| branches: | ||
| - main | ||
| - release/** | ||
| tags: | ||
| - v* | ||
| # Cancel any running jobs for PRs on a new commit | ||
| concurrency: | ||
| group: ${{ github.workflow }}-${{ github.ref }} | ||
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | ||
| jobs: | ||
| get-meta: | ||
| name: Get meta information required for the build | ||
| runs-on: ubuntu-latest | ||
| permissions: | ||
| contents: read | ||
| id-token: write | ||
| outputs: | ||
| build-type: ${{ steps.set-build-type.outputs.build-type }} | ||
| date: ${{ steps.set-meta.outputs.date }} | ||
| # For PRs we can build for all possible targets but we use a reduced set for main/releases to keep costs/time down | ||
| linux-targets: ${{ github.event_name == 'pull_request' && steps.set-meta.outputs.pr-linux-targets || steps.set-meta.outputs.main-linux-targets }} | ||
| version: ${{ steps.set-version.outputs.version }} | ||
| oss-version: ${{ steps.set-oss-version.outputs.oss-version }} | ||
| steps: | ||
| - name: Checkout code | ||
| uses: actions/checkout@v6 | ||
| - name: Debug | ||
| uses: raven-actions/debug@v1 | ||
| - name: Simple variable for type of build, i.e. release or staging | ||
| id: set-build-type | ||
| run: | | ||
| echo "Running build of type: $BUILD_TYPE" | ||
| echo "build-type=$BUILD_TYPE" >> $GITHUB_OUTPUT | ||
| shell: bash | ||
| env: | ||
| BUILD_TYPE: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/') && 'release' || 'staging' }} | ||
| - name: Get current date for nightly build info | ||
| id: set-date | ||
| run: | | ||
| if [[ '${{ steps.set-build-type.outputs.build-type }}' == 'release' ]]; then | ||
| echo "date=" >> $GITHUB_OUTPUT | ||
| else | ||
| echo "date=$(date '+%Y-%m-%d-%H_%M_%S')" >> $GITHUB_OUTPUT | ||
| fi | ||
| shell: bash | ||
| - name: Get the versions to use | ||
| # Look for the default line in the Dockerfile | ||
| id: set-version | ||
| run: | | ||
| VERSION=$(grep "ARG FLUENTDO_AGENT_VERSION=" Dockerfile.ubi | cut -d '=' -s -f 2 -) | ||
| # For releases we use the tag | ||
| if [[ '${{ steps.set-build-type.outputs.build-type }}' == 'release' ]]; then | ||
| TAG_NAME=${GITHUB_REF#refs/tags/} | ||
| echo "Found tag: $TAG_NAME" | ||
| # Strip first character | ||
| VERSION=${TAG_NAME:1} | ||
| fi | ||
| echo "Using version: $VERSION" | ||
| echo "version=$VERSION" >> $GITHUB_OUTPUT | ||
| shell: bash | ||
| - name: Extract the configuration from the JSON file | ||
| # Read from the file and remove newline characters: https://stackoverflow.com/a/64627966 | ||
| id: set-meta | ||
| run: | | ||
| PR_LINUX_TARGETS=$(cat "$JSON_FILE_NAME"|jq -c .linux_targets ) | ||
| echo "pr-linux-targets=$PR_LINUX_TARGETS" | ||
| echo "pr-linux-targets=$PR_LINUX_TARGETS" >> $GITHUB_OUTPUT | ||
| MAIN_LINUX_TARGETS=$(cat "$JSON_FILE_NAME"|jq -c .release.linux_targets ) | ||
| echo "main-linux-targets=$MAIN_LINUX_TARGETS" | ||
| echo "main-linux-targets=$MAIN_LINUX_TARGETS" >> $GITHUB_OUTPUT | ||
| shell: bash | ||
| env: | ||
| JSON_FILE_NAME: build-config.json | ||
| - name: Get the OSS version | ||
| # Read from the file and remove newline characters: https://stackoverflow.com/a/64627966 | ||
| id: set-oss-version | ||
| run: | | ||
| OSS_VERSION=$(cat source/oss_version.txt) | ||
| echo "oss-version=$OSS_VERSION" | ||
| echo "oss-version=$OSS_VERSION" >> $GITHUB_OUTPUT | ||
| shell: bash | ||
| build-image: | ||
| uses: ./.github/workflows/call-build-containers.yaml | ||
| needs: | ||
| - get-meta | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| image-base: | ||
| - "ghcr.io/fluentdo/agent/ubi" | ||
| - "ghcr.io/fluentdo/agent/debian" | ||
| permissions: | ||
| contents: read | ||
| packages: write | ||
| # This is used to complete the identity challenge | ||
| # with sigstore/fulcio when running outside of PRs. | ||
| id-token: write | ||
| with: | ||
| version: ${{ needs.get-meta.outputs.version }} | ||
| ref: ${{ github.ref }} | ||
| image-base: ${{ matrix.image-base }} | ||
| # Pick the Dockerfile to use for each image | ||
| definition: ${{ (contains(matrix.image-base, 'debian') && 'Dockerfile.debian') || 'Dockerfile.ubi' }} | ||
| dockerhub-username: ${{ vars.DOCKERHUB_USERNAME }} | ||
| # Request to build on self-hosted runners if this is a PR and we have the label set, otherwise use the standard runners | ||
| amd-runner-label: ${{ (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'build-self-hosted') && 'self-ubuntu-latest') || 'namespace-profile-ubuntu-latest' }} | ||
| # No self-hosted ARM runners yet so always use the standard ones | ||
| # arm-runner-label: "namespace-profile-ubuntu-latest-arm" | ||
| secrets: | ||
| dockerhub-token: ${{ secrets.DOCKERHUB_PUBLIC_READ_TOKEN }} | ||
| cosign_private_key: ${{ secrets.COSIGN_PRIVATE_KEY }} | ||
| cosign_private_key_password: ${{ secrets.COSIGN_PASSWORD }} | ||
| build-linux: | ||
| # Only build Linux packages if we are not a pull request or have a label set | ||
| if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'build-packages') || contains(github.event.pull_request.labels.*.name, 'build-linux') }} | ||
| uses: ./.github/workflows/call-build-linux-packages.yaml | ||
| needs: | ||
| - get-meta | ||
| with: | ||
| version: ${{ needs.get-meta.outputs.version }} | ||
| ref: ${{ github.ref }} | ||
| target-matrix: ${{ needs.get-meta.outputs.linux-targets }} | ||
| nightly-build-info: ${{ needs.get-meta.outputs.date }} | ||
| dockerhub-username: ${{ vars.DOCKERHUB_USERNAME }} | ||
| # Request to build on self-hosted runners if this is a PR and we have the label set, otherwise use the standard runners | ||
| amd-runner-label: ${{ (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'build-self-hosted') && 'self-ubuntu-latest') || 'namespace-profile-ubuntu-latest' }} | ||
| large-amd-runner-label: ${{ (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'build-self-hosted') && 'self-ubuntu-latest') || 'namespace-profile-ubuntu-latest-4cpu-16gb' }} | ||
| # No self-hosted ARM runners yet so always use the standard ones | ||
| # arm-runner-label: "namespace-profile-ubuntu-latest-arm" | ||
| secrets: | ||
| dockerhub-token: ${{ secrets.DOCKERHUB_PUBLIC_READ_TOKEN }} | ||
| build-windows: | ||
| # Only build Windows packages if we are not a pull request or have a label set | ||
| if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'build-packages') || contains(github.event.pull_request.labels.*.name, 'build-windows') }} | ||
| uses: ./.github/workflows/call-build-windows-packages.yaml | ||
| needs: | ||
| - get-meta | ||
| with: | ||
| version: ${{ needs.get-meta.outputs.version }} | ||
| ref: ${{ github.ref }} | ||
| nightly-build-info: ${{ needs.get-meta.outputs.date }} | ||
| build-macos: | ||
| # Only build macOS packages if we are not a pull request or have a label set | ||
| # if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'build-packages') || contains(github.event.pull_request.labels.*.name, 'build-macos') }} | ||
| # For now we only build for PRs with the label to reduce unnecessary costs with macOS runners | ||
| if: ${{ github.event_name == 'pull_request' && (contains(github.event.pull_request.labels.*.name, 'build-packages') || contains(github.event.pull_request.labels.*.name, 'build-macos')) }} | ||
| uses: ./.github/workflows/call-build-macos-packages.yaml | ||
| needs: | ||
| - get-meta | ||
| with: | ||
| version: ${{ needs.get-meta.outputs.version }} | ||
| ref: ${{ github.ref }} | ||
| nightly-build-info: ${{ needs.get-meta.outputs.date }} | ||
| # We want to copy the UBI image to ghcr.io/fluentdo/agent:version and | ||
| # the distroless image to ghcr.io/fluentdo/agent:version-slim | ||
| copy-common-images: | ||
| # Only run for releases to reduce unnecessary load on the registry | ||
| if: ${{ startsWith(github.ref, 'refs/tags/v') }} | ||
| name: Promote release images | ||
| needs: | ||
| - get-meta | ||
| - build-image | ||
| permissions: | ||
| contents: read | ||
| packages: write | ||
| id-token: write | ||
| uses: ./.github/workflows/call-publish-release-images.yaml | ||
| with: | ||
| version: ${{ needs.build-image.outputs.version }} | ||
| secrets: | ||
| github-token: ${{ secrets.GITHUB_TOKEN }} | ||
| test-containers: | ||
| name: Test container images | ||
| uses: ./.github/workflows/call-test-containers.yaml | ||
| with: | ||
| image: ${{ matrix.image-base }} | ||
| image-tag: ${{ needs.build-image.outputs.version }} | ||
| ref: ${{ github.ref }} | ||
| secrets: | ||
| github-token: ${{ secrets.GITHUB_TOKEN }} | ||
| needs: | ||
| - build-image | ||
| permissions: | ||
| packages: read | ||
| contents: read | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| image-base: | ||
| - "ghcr.io/fluentdo/agent/ubi" | ||
| - "ghcr.io/fluentdo/agent/debian" | ||
| test-packages: | ||
| name: Test packages | ||
| needs: | ||
| - get-meta | ||
| - build-linux | ||
| uses: ./.github/workflows/call-test-packages.yaml | ||
| with: | ||
| build-matrix: ${{ needs.get-meta.outputs.linux-targets }} | ||
| version: ${{ needs.get-meta.outputs.version }} | ||
| ref: ${{ github.ref }} | ||
| dockerhub-username: ${{ vars.DOCKERHUB_USERNAME }} | ||
| # Request to run on self-hosted runners if this is a PR and we have the label set, otherwise use the standard runners | ||
| amd-runner-label: ${{ (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'build-self-hosted') && 'self-ubuntu-latest') || 'namespace-profile-ubuntu-latest' }} | ||
| secrets: | ||
| dockerhub-token: ${{ secrets.DOCKERHUB_PUBLIC_READ_TOKEN }} | ||
| github-token: ${{ secrets.GITHUB_TOKEN }} | ||
| # Placeholder to make it simple to create a status check on this, do not change name. | ||
| # Instead of modifying branch protection rules every time we add a new job, we just | ||
| # add it as a dependency of this job. | ||
| # This job must always run last so it depends on all other jobs that must complete. | ||
| # Note that jobs that are conditionally run (e.g. not on PRs) must be excluded here. | ||
| # If you add a new job that must complete then add it here. | ||
| tests-complete: | ||
| name: All tests complete | ||
| # We use this to always run even if a previous job fails or is skipped (which we check for) | ||
| if: always() | ||
| needs: | ||
| - build-linux | ||
| - build-windows | ||
| - build-macos | ||
| - build-image | ||
| - test-containers | ||
| # TODO: remove once failures are resolved to ensure auto-release job still goes ahead with latest commit | ||
| # https://github.com/FluentDo/agent/issues/73 | ||
| # - test-packages | ||
| # Add additional jobs here as required that must complete | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Decide whether the needed jobs succeeded or failed | ||
| uses: re-actors/alls-green@release/v1 | ||
| with: | ||
| # Add any jobs that can be skipped here to avoid failure of this job | ||
| allowed-skips: build-linux,build-windows,build-macos,test-packages | ||
| # Convert the needs object to JSON to pass it in | ||
| jobs: ${{ toJSON(needs) }} | ||
| - name: All tests complete | ||
| run: echo "All tests complete" | ||
| shell: bash | ||
| # runs for every commit except only versioned releases are pushed. this allows dry running the job for every PR and | ||
| # allows dependent downstream jobs to dry run as well. | ||
| release: | ||
| name: Create release | ||
| needs: | ||
| - get-meta | ||
| - build-image | ||
| # We need packages to be built for a release | ||
| - build-linux | ||
| - build-windows | ||
| # Deliberately exclude the following for a release and also to dry run PRs | ||
| # - copy-common-images | ||
| # - tests-complete | ||
| runs-on: ubuntu-latest | ||
| permissions: | ||
| packages: read | ||
| actions: read | ||
| contents: write | ||
| id-token: write | ||
| env: | ||
| UBI_IMAGE_NAME: ghcr.io/fluentdo/agent/ubi | ||
| DISTROLESS_IMAGE_NAME: ghcr.io/fluentdo/agent/debian | ||
| TAG: ${{ needs.build-image.outputs.version }} | ||
| steps: | ||
| - uses: actions/checkout@v6 | ||
| # We need full history to be able to detect branch ancestry | ||
| with: | ||
| fetch-depth: 0 | ||
| - name: Download all artefacts | ||
| uses: actions/download-artifact@v7 | ||
| with: | ||
| path: output | ||
| pattern: | | ||
| *package* | ||
| - name: Filter out headers and extra packages | ||
| run: | | ||
| find output/ -type f \( -iname '*-headers.rpm' -o -iname '*-extra.rpm' -o -iname '*-headers.deb' -o -iname '*-extra.deb' \) -delete -print | ||
| shell: bash | ||
| - name: Install dependencies to support signing | ||
| run: | | ||
| sudo apt-get update | ||
| sudo apt-get install -y createrepo-c rpm debsigs coreutils | ||
| shell: bash | ||
| - name: Authenticate with GCP | ||
| uses: google-github-actions/auth@v3 | ||
| with: | ||
| workload_identity_provider: "projects/841522437311/locations/global/workloadIdentityPools/github-actions/providers/github-actions" | ||
| service_account: "terraform-infra@infrastructure-464010.iam.gserviceaccount.com" | ||
| - id: get-secrets | ||
| name: Get secrets from GCP Secret Manager | ||
| # This step retrieves secrets from GCP Secret Manager and sets them as outputs | ||
| # The secrets can then be accessed in subsequent steps using ${{ steps.get-secrets.outputs.<secret_name> }} | ||
| uses: "google-github-actions/get-secretmanager-secrets@v3" | ||
| with: | ||
| secrets: |- | ||
| gpg_private_key:projects/626836145334/secrets/GPG_PRIVATE_KEY | ||
| gpg_private_key_passphrase:projects/626836145334/secrets/GPG_PRIVATE_KEY_PASSPHRASE | ||
| - name: Import GPG key for signing | ||
| id: import_gpg | ||
| uses: crazy-max/ghaction-import-gpg@v6 | ||
| with: | ||
| gpg_private_key: ${{ steps.get-secrets.outputs.gpg_private_key }} | ||
| passphrase: ${{ steps.get-secrets.outputs.gpg_private_key_passphrase }} | ||
| - name: Sign and generate checksums | ||
| run: ./scripts/sign-packages.sh | ||
| env: | ||
| GPG_KEY: ${{ steps.import_gpg.outputs.name }} | ||
| BASE_DIR: ${{ github.workspace }}/output/ | ||
| - name: Package into a single tarball so we can have individual distributions | ||
| # No packages unless we use specific labels for a PR | ||
| if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'build-packages') || contains(github.event.pull_request.labels.*.name, 'build-linux') || contains(github.event.pull_request.labels.*.name, 'build-windows') }} | ||
| run: | | ||
| tar -czvf $GITHUB_WORKSPACE/deliverables.tar.gz -C output . | ||
| shell: bash | ||
| - name: Log in to the Container registry | ||
| uses: docker/login-action@v3 | ||
| with: | ||
| registry: ghcr.io | ||
| username: ${{ github.actor }} | ||
| password: ${{ secrets.GITHUB_TOKEN }} | ||
| - uses: anchore/sbom-action@v0 | ||
| with: | ||
| image: ${{ env.UBI_IMAGE_NAME }}:${{ env.TAG }} | ||
| artifact-name: image-ubi-sbom.spdx | ||
| registry-username: ${{ github.actor }} | ||
| registry-password: ${{ secrets.GITHUB_TOKEN }} | ||
| output-file: image-ubi-sbom.spdx | ||
| - uses: anchore/sbom-action@v0 | ||
| with: | ||
| image: ${{ env.DISTROLESS_IMAGE_NAME }}:${{ env.TAG }} | ||
| artifact-name: image-debian-sbom.spdx | ||
| registry-username: ${{ github.actor }} | ||
| registry-password: ${{ secrets.GITHUB_TOKEN }} | ||
| output-file: image-debian-sbom.spdx | ||
| - name: Save image as tarball | ||
| run: | | ||
| mkdir -p output | ||
| skopeo copy \ | ||
| --all \ | ||
| --remove-signatures \ | ||
| --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ | ||
| "docker://${{ env.UBI_IMAGE_NAME }}:${{ env.TAG }}" \ | ||
| "oci-archive:output/fluentdo-agent-container.tar" | ||
| skopeo copy \ | ||
| --all \ | ||
| --remove-signatures \ | ||
| --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ | ||
| "docker://${{ env.DISTROLESS_IMAGE_NAME }}:${{ env.TAG }}" \ | ||
| "oci-archive:output/fluentdo-agent-container-slim.tar" | ||
| tar -czvf output/fluentdo-agent-container.tar.gz output/fluentdo-agent-container.tar output/fluentdo-agent-container-slim.tar | ||
| rm -f output/fluentdo-agent-container.tar output/fluentdo-agent-container-slim.tar | ||
| shell: bash | ||
| - name: Construct release info | ||
| # Add target info and OSS version to new JSON file | ||
| run: | | ||
| jq '. += { "oss_version": "${{ needs.get-meta.outputs.oss-version }}"}' build-config.json | tee output/release.json | ||
| shell: bash | ||
| - name: Make nice Linux targets list | ||
| id: linux-targets | ||
| run: | | ||
| LINUX_TARGETS=$(echo '${{ needs.get-meta.outputs.linux-targets }}' | jq -r '. | join(", ")') | ||
| echo "linux-targets=$LINUX_TARGETS" | ||
| echo "linux-targets=$LINUX_TARGETS" >> $GITHUB_OUTPUT | ||
| shell: bash | ||
| - name: Debug | ||
| if: always() | ||
| run: ls -lR | ||
| shell: bash | ||
| - name: Check if commit is on default branch | ||
| id: check_branch | ||
| run: | | ||
| # Check if the current commit exists on the default branch | ||
| if git branch -r --contains ${{ github.sha }} | grep -q "origin/${{ github.event.repository.default_branch }}"; then | ||
| echo "is_default=true" >> $GITHUB_OUTPUT | ||
| else | ||
| echo "is_default=false" >> $GITHUB_OUTPUT | ||
| fi | ||
| shell: bash | ||
| - name: Create release | ||
| if: ${{ startsWith(github.ref, 'refs/tags/v') }} | ||
| # This may fail for workflow_dispatch if the release already exists | ||
| uses: softprops/action-gh-release@v2 | ||
| with: | ||
| append_body: true | ||
| body: | | ||
| Fluent Do Agent release for ${{ github.ref_name }} version | ||
| Version: ${{ needs.get-meta.outputs.version }} | ||
| OSS Version: ${{ needs.get-meta.outputs.oss-version }} | ||
| Targets: | ||
| - Windows: 2022/2025 | ||
| - Linux: ${{ steps.linux-targets.outputs.linux-targets }} | ||
| Images: | ||
| - ${{ env.UBI_IMAGE_NAME }}:${{ env.TAG }} | ||
| - ${{ env.DISTROLESS_IMAGE_NAME }}:${{ env.TAG }} | ||
| files: | | ||
| install.sh | ||
| *.spdx | ||
| output/*.json | ||
| deliverables.tar.gz | ||
| output/fluentdo-agent-container.tar.gz | ||
| output/package-macos-* | ||
| output/package-windows-* | ||
| fail_on_unmatched_files: false | ||
| generate_release_notes: true | ||
| tag_name: ${{ github.ref_name }} | ||
| # Ensure we create from the appropriate ref for non-latest builds | ||
| # Use the commit SHA to ensure we create from the exact commit the tag points to | ||
| target_commitish: ${{ github.sha }} | ||
| # Only make the latest release if the commit exists on the default branch | ||
| make_latest: ${{ steps.check_branch.outputs.is_default == 'true' }} | ||
| - name: Upload packages to Google Cloud Storage if not a PR | ||
| # Only if not a pull request | ||
| if: ${{ github.event_name != 'pull_request' }} | ||
| id: upload-packages | ||
| uses: google-github-actions/upload-cloud-storage@v3 | ||
| with: | ||
| path: output/ | ||
| destination: fluentdo-agent-${{ needs.get-meta.outputs.build-type }}/${{ needs.get-meta.outputs.version }}/ | ||
| update-docs: | ||
| needs: | ||
| - get-meta | ||
| - release | ||
| if: ${{ startsWith(github.ref, 'refs/tags/v') }} | ||
| name: Update documentation on release | ||
| uses: fluentdo/documentation/.github/workflows/call-add-mapping-version.yaml@main | ||
| with: | ||
| agent-version: ${{ needs.get-meta.outputs.version }} | ||
| oss-version: ${{ needs.get-meta.outputs.oss-version }} | ||
| update-homebrew: | ||
| name: Update homebrew on release | ||
| if: ${{ startsWith(github.ref, 'refs/tags/v') }} | ||
| needs: | ||
| - get-meta | ||
| - release | ||
| - build-macos | ||
| uses: fluentdo/homebrew-fluentdo-agent/.github/workflows/call-update-homebrew.yaml@main | ||
|
Check failure on line 485 in .github/workflows/build.yaml
|
||
| with: | ||
| version: ${{ github.ref_name }} | ||
| secrets: | ||
| github-token: ${{ secrets.GITHUB_TOKEN }} | ||
