add various suggested changes from code review #6
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-License-Identifier: Apache-2.0 | ||
|
Check failure on line 1 in .github/workflows/zxc-verify-docker-build-determinism.yaml
|
||
| name: "ZXC: Verify Docker Build Determinism" | ||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| ref: | ||
| description: "The branch, tag, or commit to checkout:" | ||
| type: string | ||
| required: false | ||
| default: "" | ||
| java-distribution: | ||
| description: "Java JDK Distribution:" | ||
| type: string | ||
| required: false | ||
| default: "temurin" | ||
| java-version: | ||
| description: "Java JDK Version:" | ||
| type: string | ||
| required: false | ||
| default: "25" | ||
| secrets: | ||
| github-token: | ||
| description: "GitHub Token with permissions to checkout the repository." | ||
| required: true | ||
| gradle-cache-username: | ||
| description: "The username used to authenticate with the Gradle Build Cache Node." | ||
| required: true | ||
| gradle-cache-password: | ||
| description: "The password used to authenticate with the Gradle Build Cache Node." | ||
| required: true | ||
| outputs: | ||
| failure-mode: | ||
| description: "Failure Mode" | ||
| value: ${{ jobs.set-failure-mode.outputs.failure-mode }} | ||
| defaults: | ||
| run: | ||
| shell: bash | ||
| permissions: | ||
| id-token: write | ||
| contents: read | ||
| env: | ||
| GRADLE_CACHE_USERNAME: ${{ secrets.gradle-cache-username }} | ||
| GRADLE_CACHE_PASSWORD: ${{ secrets.gradle-cache-password }} | ||
| DOCKER_MANIFEST_GENERATOR: .github/workflows/support/scripts/generate-docker-artifact-baseline.sh | ||
| DOCKER_MANIFEST_PATH: ${{ github.workspace }}/.manifests/docker | ||
| DOCKER_REGISTRY: localhost:5000 | ||
| DOCKER_IMAGE_NAME: consensus-node | ||
| DOCKER_CONTEXT_PATH: hedera-node/infrastructure/docker/containers/production-next/consensus-node | ||
| SKOPEO_VERSION: v1.14.0 | ||
| jobs: | ||
| generate-baseline: | ||
| name: "Docker: Generate Baseline" | ||
| runs-on: hl-cn-docker-determinism-lin-md | ||
| outputs: | ||
| sha: ${{ steps.commit.outputs.sha }} | ||
| sha-abbrev: ${{ steps.commit.outputs.sha-abbrev }} | ||
| source-date: ${{ steps.commit.outputs.source-date }} | ||
| path: ${{ steps.baseline.outputs.path }} | ||
| file: ${{ steps.baseline.outputs.file }} | ||
| name: ${{ steps.baseline.outputs.name }} | ||
| failure-mode: ${{ steps.set-failure-mode.outputs.failure-mode }} | ||
| steps: | ||
| - name: Prepare Job | ||
| uses: pandaswhocode/initialize-github-job@a57cd6d8d768b2f3c95334bdd4fa8c21609fc651 # v1.0.5 | ||
| with: | ||
| checkout: "true" | ||
| checkout-ref: "${{ inputs.ref }}" | ||
| checkout-token: "${{ secrets.github-token }}" | ||
| checkout-fetch-depth: "1" | ||
| setup-java: "true" | ||
| java-distribution: "${{ inputs.java-distribution }}" | ||
| java-version: "${{ inputs.java-version }}" | ||
| setup-gradle: "true" | ||
| gradle-cache-read-only: "false" | ||
| - name: Authenticate to Google Cloud | ||
| id: google-auth | ||
| uses: step-security/google-github-auth@40f6deebd366f16c782d7a0ad0844e3b96a032a6 # v2.1.10 | ||
| with: | ||
| workload_identity_provider: "projects/235822363393/locations/global/workloadIdentityPools/hedera-builds-pool/providers/hedera-builds-gh-actions" | ||
| service_account: "swirlds-automation@hedera-registry.iam.gserviceaccount.com" | ||
| - name: Setup Google Cloud SDK | ||
| uses: google-github-actions/setup-gcloud@77e7a554d41e2ee56fc945c52dfd3f33d12def9a # v2.1.4 | ||
| - name: Retrieve Commit Hash | ||
| id: commit | ||
| run: | | ||
| echo "sha=$(git rev-parse HEAD)" >> "${GITHUB_OUTPUT}" | ||
| echo "sha-abbrev=$(git rev-parse HEAD | tr -d '[:space:]' | cut -c1-8)" >> "${GITHUB_OUTPUT}" | ||
| echo "source-date=$(git log -1 --pretty=%ct)" >> "${GITHUB_OUTPUT}" | ||
| - name: Baseline Existence Check | ||
| id: baseline | ||
| run: | | ||
| BASELINE_NAME="${{ steps.commit.outputs.sha }}.tar.gz" | ||
| BASELINE_PATH="gs://hedera-ci-ephemeral-artifacts/${{ github.repository }}/docker/baselines" | ||
| BASELINE_FILE="${BASELINE_PATH}/${BASELINE_NAME}" | ||
| BASELINE_EXISTS="false" | ||
| if gsutil ls "${BASELINE_FILE}" >/dev/null 2>&1; then | ||
| BASELINE_EXISTS="true" | ||
| fi | ||
| echo "exists=${BASELINE_EXISTS}" >> "${GITHUB_OUTPUT}" | ||
| echo "path=${BASELINE_PATH}" >> "${GITHUB_OUTPUT}" | ||
| echo "name=${BASELINE_NAME}" >> "${GITHUB_OUTPUT}" | ||
| echo "file=${BASELINE_FILE}" >> "${GITHUB_OUTPUT}" | ||
| - name: Install Skopeo and JQ | ||
| if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} | ||
| run: | | ||
| sudo apt-get update | ||
| sudo apt-get install --yes --no-install-recommends skopeo jq | ||
| - name: Setup QEmu Support | ||
| uses: docker/setup-qemu-action@5964de0df58d5ad28b04d8fe2e6b80ad47105b91 # v3.5.0 | ||
| if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} | ||
| - name: Setup Docker Buildx Support | ||
| uses: step-security/setup-buildx-action@f931205d68723ad9589fd2a7e2ece238bf9de341 # v4.0.0 | ||
| if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} | ||
| with: | ||
| version: v0.32.1 | ||
| driver-opts: network=host,image=ghcr.io/hashgraph/runner-images/buildkit:buildx-stable-1 | ||
| buildkitd-config-inline: | | ||
| [registry."docker.io"] | ||
| mirrors = ["https://hub.mirror.docker.lat.ope.eng.hashgraph.io"] | ||
| - name: Setup Local Docker Registry | ||
| if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} | ||
| run: docker run -d -p 5000:5000 --restart=always --name registry registry:latest | ||
| - name: Show Docker Version | ||
| if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} | ||
| run: docker version | ||
| - name: Show Docker Info | ||
| if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} | ||
| run: docker info | ||
| - name: Build Gradle Artifacts | ||
| if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} | ||
| run: ./gradlew assemble | ||
| - name: Prepare for Docker Build | ||
| if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} | ||
| run: | | ||
| mkdir -p "${{ github.workspace }}/${{ env.DOCKER_CONTEXT_PATH }}/sdk/data" | ||
| echo "::group::Copying Library Artifacts" | ||
| cp -Rvf "${{ github.workspace }}/hedera-node/data/lib" "${{ github.workspace }}/${{ env.DOCKER_CONTEXT_PATH }}/sdk/data/" | ||
| echo "::endgroup::" | ||
| echo "::group::Copying Application Artifacts" | ||
| cp -Rvf "${{ github.workspace }}/hedera-node/data/apps" "${{ github.workspace }}/${{ env.DOCKER_CONTEXT_PATH }}/sdk/data/" | ||
| echo "::endgroup::" | ||
| - name: Write Artifact Version Descriptor | ||
| if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} | ||
| run: | | ||
| printf "VERSION=%s\nCOMMIT=%s\nDATE=%s" "$(./gradlew -q showVersion)" "$(git log -1 --format='%H' | cut -c1-8)" "$(date -u)" | tee "${{ github.workspace }}/${{ env.DOCKER_CONTEXT_PATH }}/sdk/VERSION" | ||
| - name: Build Docker Image | ||
| uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 | ||
| env: | ||
| SOURCE_DATE_EPOCH: ${{ steps.commit.outputs.source-date }} | ||
| if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} | ||
| with: | ||
| push: true | ||
| no-cache: true | ||
| platforms: linux/amd64,linux/arm64 | ||
| build-args: | | ||
| SOURCE_DATE_EPOCH=${{ steps.commit.outputs.source-date }} | ||
| context: ${{ env.DOCKER_CONTEXT_PATH }} | ||
| tags: ${{ env.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE_NAME }}:${{ steps.commit.outputs.sha-abbrev }} | ||
| - name: Generate Manifest | ||
| id: manifest | ||
| env: | ||
| MANIFEST_PATH: ${{ env.DOCKER_MANIFEST_PATH }} | ||
| if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} | ||
| run: GITHUB_SHA="${{ needs.generate-baseline.outputs.sha-abbrev }}" ${{ env.DOCKER_MANIFEST_GENERATOR }} | ||
| - name: Amend Manifest with Gradle Artifacts | ||
| if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} | ||
| working-directory: ${{ env.DOCKER_MANIFEST_PATH }} | ||
| run: | | ||
| EXTRACTED_FILE_NAME="${{ steps.commit.outputs.sha }}.tar" | ||
| gunzip "${{ steps.manifest.outputs.name }}" | ||
| tar -rvf "${EXTRACTED_FILE_NAME}" -C "${{ github.workspace }}/${{ env.DOCKER_CONTEXT_PATH }}" sdk | ||
| gzip "${EXTRACTED_FILE_NAME}" | ||
| - name: Upload Baseline | ||
| if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }} | ||
| run: gsutil cp "${{ steps.manifest.outputs.file }}" "${{ steps.baseline.outputs.file }}" | ||
| - name: Set Failure Mode Output | ||
| id: set-failure-mode | ||
| if: ${{ always() }} | ||
| run: | | ||
| if ${{ success() }}; then | ||
| echo "failure-mode=none" >> "${GITHUB_OUTPUT}" | ||
| else | ||
| echo "failure-mode=workflow" >> "${GITHUB_OUTPUT}" | ||
| fi | ||
| verify-artifacts: | ||
| name: "Docker: Verify Artifacts (${{ join(matrix.os, ', ') }})" | ||
| runs-on: ${{ matrix.os }} | ||
| needs: | ||
| - generate-baseline | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| # Windows is not supported due to GitHub not supporting Docker Desktop/Podman Desktop and Docker CE on Windows | ||
| # not supporting BuildKit and the Buildx plugin. | ||
| # GitHub hosted MacOS and Ubuntu runners are temporarily disabled. | ||
| os: | ||
| - ubuntu-24.04 | ||
| #- ubuntu-22.04 | ||
| #- macos-12 | ||
| #- macos-11 | ||
| #- hl-cn-docker-determinism-lin-md | ||
| - hl-cn-docker-determinism-lin-lg | ||
| - hl-cn-docker-determinism-lin-u24-lg | ||
| - hl-cn-docker-determinism-lin-d12-lg | ||
| outputs: | ||
| failure-mode: ${{ steps.set-failure-mode.outputs.failure-mode }} | ||
| steps: | ||
| - name: Standardize Git Line Endings | ||
| run: | | ||
| git config --global core.autocrlf false | ||
| git config --global core.eol lf | ||
| - name: Prepare Job | ||
| uses: pandaswhocode/initialize-github-job@a57cd6d8d768b2f3c95334bdd4fa8c21609fc651 # v1.0.5 | ||
| with: | ||
| checkout: "true" | ||
| checkout-ref: "${{ inputs.ref }}" | ||
| checkout-token: "${{ secrets.github-token }}" | ||
| checkout-fetch-depth: "1" | ||
| setup-java: "true" | ||
| java-distribution: "${{ inputs.java-distribution }}" | ||
| java-version: "${{ inputs.java-version }}" | ||
| setup-gradle: "true" | ||
| gradle-cache-read-only: "false" | ||
| - name: Install Python (Linux) | ||
| if: ${{ runner.os == 'Linux' }} | ||
| run: | | ||
| sudo apt-get update | ||
| sudo apt-get install --yes --no-install-recommends python3 python3-venv python3-pip | ||
| - name: Install Python | ||
| if: ${{ runner.os != 'Linux' }} | ||
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | ||
| with: | ||
| python-version: "3.12" | ||
| - name: Install JQ (Linux) | ||
| if: ${{ runner.os == 'Linux' }} | ||
| run: | | ||
| sudo apt-get update | ||
| sudo apt-get install --yes --no-install-recommends jq | ||
| - name: Install Skopeo (Linux) | ||
| if: ${{ runner.os == 'Linux' }} | ||
| run: | | ||
| source /etc/os-release | ||
| if [[ "${VERSION_ID}" != "20.04" ]]; then | ||
| sudo apt-get install --yes --no-install-recommends skopeo | ||
| fi | ||
| - name: Install Skopeo and JQ (macOS) | ||
| if: ${{ runner.os == 'macOS' }} | ||
| run: brew install skopeo jq | ||
| - name: Setup CoreUtils (macOS) | ||
| if: ${{ runner.os == 'macOS' }} | ||
| run: brew install coreutils | ||
| - name: Authenticate to Google Cloud | ||
| id: google-auth | ||
| uses: step-security/google-github-auth@40f6deebd366f16c782d7a0ad0844e3b96a032a6 # v2.1.10 | ||
| with: | ||
| workload_identity_provider: "projects/235822363393/locations/global/workloadIdentityPools/hedera-builds-pool/providers/hedera-builds-gh-actions" | ||
| service_account: "swirlds-automation@hedera-registry.iam.gserviceaccount.com" | ||
| - name: Setup Google Cloud SDK | ||
| uses: google-github-actions/setup-gcloud@77e7a554d41e2ee56fc945c52dfd3f33d12def9a # v2.1.4 | ||
| env: | ||
| CLOUDSDK_PYTHON: ${{ format('{0}{1}', env.pythonLocation, runner.os == 'Windows' && '\python.exe' || '/bin/python3') }} | ||
| - name: Download Baseline | ||
| env: | ||
| CLOUDSDK_PYTHON: ${{ format('{0}{1}', env.pythonLocation, runner.os == 'Windows' && '\python.exe' || '/bin/python3') }} | ||
| run: | | ||
| mkdir -p "${DOCKER_MANIFEST_PATH}" | ||
| cd "${DOCKER_MANIFEST_PATH}" | ||
| gsutil cp "${{ needs.generate-baseline.outputs.file }}" . | ||
| tar -xzf "${{ needs.generate-baseline.outputs.name }}" | ||
| mv "sdk" "${{ github.workspace }}/${{ env.DOCKER_CONTEXT_PATH }}/" | ||
| - name: Remove Preinstalled Docker | ||
| if: ${{ runner.os == 'macOS' }} | ||
| run: | | ||
| set -x | ||
| sudo killall dockerd || true | ||
| sudo killall containerd || true | ||
| sudo rm -rvf /usr/bin/*containerd* || true | ||
| sudo rm -rvf /usr/bin/docker* || true | ||
| sudo rm -rvf /usr/local/bin/docker* || true | ||
| sudo rm -rvf /usr/local/bin/*lima* || true | ||
| - name: Install Lima (macOS) | ||
| if: ${{ runner.os == 'macOS' }} | ||
| run: | | ||
| VERSION="v0.20.0" | ||
| curl -fsSL "https://github.com/lima-vm/lima/releases/download/${VERSION}/lima-${VERSION:1}-$(uname -s)-$(uname -m).tar.gz" | sudo tar Cxzvm /usr/local | ||
| - name: Determine Home Directory | ||
| id: home | ||
| run: echo "directory=$(tr -d '[:space:]' < <(cd ~ && pwd))" >> "${GITHUB_OUTPUT}" | ||
| - name: Setup QEmu Support | ||
| uses: docker/setup-qemu-action@5964de0df58d5ad28b04d8fe2e6b80ad47105b91 # v3.5.0 | ||
| - name: Setup Docker Buildx Support | ||
| uses: step-security/setup-buildx-action@f931205d68723ad9589fd2a7e2ece238bf9de341 # v4.0.0 | ||
| with: | ||
| version: v0.32.1 | ||
| driver-opts: network=host,image=ghcr.io/hashgraph/runner-images/buildkit:buildx-stable-1 | ||
| buildkitd-config-inline: | | ||
| [registry."docker.io"] | ||
| mirrors = ["https://hub.mirror.docker.lat.ope.eng.hashgraph.io"] | ||
| - name: Setup Local Docker Registry | ||
| run: docker run -d -p 5000:5000 --restart=always --name registry registry:latest | ||
| - name: Show Docker Version | ||
| run: docker version | ||
| - name: Show Docker Info | ||
| run: docker info | ||
| - name: Build Docker Image | ||
| uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 | ||
| env: | ||
| SOURCE_DATE_EPOCH: ${{ needs.generate-baseline.outputs.source-date }} | ||
| with: | ||
| push: true | ||
| no-cache: true | ||
| platforms: linux/amd64,linux/arm64 | ||
| build-args: | | ||
| SOURCE_DATE_EPOCH=${{ needs.generate-baseline.outputs.source-date }} | ||
| context: ${{ env.DOCKER_CONTEXT_PATH }} | ||
| tags: ${{ env.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE_NAME }}:${{ needs.generate-baseline.outputs.sha-abbrev }} | ||
| - name: Regenerate Manifest | ||
| id: regen-manifest | ||
| env: | ||
| MANIFEST_PATH: ${{ env.DOCKER_MANIFEST_PATH }}/regenerated | ||
| run: GITHUB_SHA="${{ needs.generate-baseline.outputs.sha-abbrev }}" ${{ env.DOCKER_MANIFEST_GENERATOR }} | ||
| - name: Validate Layers (linux/amd64) | ||
| run: | | ||
| if ! diff -u "${DOCKER_MANIFEST_PATH}/linux-amd64.layers.json" "${{ steps.regen-manifest.outputs.path }}/linux-amd64.layers.json" >/dev/null 2>&1; then | ||
| echo "::group::Layer Differences" | ||
| diff -u "${DOCKER_MANIFEST_PATH}/linux-amd64.layers.json" "${{ steps.regen-manifest.outputs.path }}/linux-amd64.layers.json" | ||
| echo "::endgroup::" | ||
| exit 1 | ||
| fi | ||
| - name: Validate Layers (linux/arm64) | ||
| run: | | ||
| if ! diff -u "${DOCKER_MANIFEST_PATH}/linux-arm64.layers.json" "${{ steps.regen-manifest.outputs.path }}/linux-arm64.layers.json" >/dev/null 2>&1; then | ||
| echo "::group::Layer Differences" | ||
| diff -u "${DOCKER_MANIFEST_PATH}/linux-arm64.layers.json" "${{ steps.regen-manifest.outputs.path }}/linux-arm64.layers.json" | ||
| echo "::endgroup::" | ||
| exit 1 | ||
| fi | ||
| - name: Validate Full Manifest (linux/amd64) | ||
| run: | | ||
| if ! diff -u "${DOCKER_MANIFEST_PATH}/linux-amd64.comparable.json" "${{ steps.regen-manifest.outputs.path }}/linux-amd64.comparable.json" >/dev/null 2>&1; then | ||
| echo "::group::Layer Differences" | ||
| diff -u "${DOCKER_MANIFEST_PATH}/linux-amd64.comparable.json" "${{ steps.regen-manifest.outputs.path }}/linux-amd64.comparable.json" | ||
| echo "::endgroup::" | ||
| exit 1 | ||
| fi | ||
| - name: Validate Full Manifest (linux/arm64) | ||
| run: | | ||
| if ! diff -u "${DOCKER_MANIFEST_PATH}/linux-arm64.comparable.json" "${{ steps.regen-manifest.outputs.path }}/linux-arm64.comparable.json" >/dev/null 2>&1; then | ||
| echo "::group::Layer Differences" | ||
| diff -u "${DOCKER_MANIFEST_PATH}/linux-arm64.comparable.json" "${{ steps.regen-manifest.outputs.path }}/linux-arm64.comparable.json" | ||
| echo "::endgroup::" | ||
| exit 1 | ||
| fi | ||
| - name: Publish Manifests | ||
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | ||
| if: ${{ steps.regen-manifest.conclusion == 'success' && failure() && !cancelled() }} | ||
| with: | ||
| name: Docker Manifests [${{ join(matrix.os, ', ') }}] | ||
| path: ${{ env.DOCKER_MANIFEST_PATH }}/** | ||
| - name: Set Failure Mode Output | ||
| id: set-failure-mode | ||
| if: ${{ always() }} | ||
| run: | | ||
| # If the job overall succeeded, mark as 'none', otherwise 'workflow'. | ||
| if ${{ success() }}; then | ||
| echo "failure-mode=none" >> "${GITHUB_OUTPUT}" | ||
| else | ||
| echo "failure-mode=workflow" >> "${GITHUB_OUTPUT}" | ||
| fi | ||
| set-failure-mode: | ||
| name: "Set Failure Mode" | ||
| runs-on: hl-cn-default-lin-sm | ||
| needs: | ||
| - generate-baseline | ||
| - verify-artifacts | ||
| if: ${{ !cancelled() && always() }} | ||
| outputs: | ||
| failure-mode: ${{ steps.set-failure-mode.outputs.failure-mode }} | ||
| steps: | ||
| - name: Harden Runner | ||
| uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0 | ||
| with: | ||
| egress-policy: audit | ||
| - name: Set Failure Mode | ||
| id: set-failure-mode | ||
| run: | | ||
| job_statuses=( | ||
| "${{ needs.generate-baseline.result || 'skipped' }}" | ||
| "${{ needs.verify-artifacts.result || 'skipped' }}" | ||
| ) | ||
| job_failure_modes=( | ||
| "${{ needs.generate-baseline.outputs.failure-mode || 'none' }}" | ||
| "${{ needs.verify-artifacts.outputs.failure-mode || 'none' }}" | ||
| ) | ||
| failure_mode="none" | ||
| for i in "${!job_statuses[@]}"; do | ||
| if [[ "${job_statuses[$i]}" == "failure" ]]; then | ||
| mode="${job_failure_modes[$i]}" | ||
| if [[ "${failure_mode}" == "none" ]]; then | ||
| failure_mode="${mode}" | ||
| elif [[ "${mode}" == "workflow" ]]; then | ||
| failure_mode="workflow" | ||
| fi | ||
| fi | ||
| done | ||
| echo "failure-mode=${failure_mode}" >> "${GITHUB_OUTPUT}" | ||
