zxc-verify-docker-build-determinism.yaml

# SPDX-License-Identifier: Apache-2.0
name: "ZXC: Verify Docker Build Determinism"
on:
  workflow_call:
    inputs:
      ref:
        description: "The branch, tag, or commit to checkout:"
        type: string
        required: false
        default: ""
      java-distribution:
        description: "Java JDK Distribution:"
        type: string
        required: false
        default: "temurin"
      java-version:
        description: "Java JDK Version:"
        type: string
        required: false
        default: "25"

    secrets:
      github-token:
        description: "GitHub Token with permissions to checkout the repository."
        required: true
      gradle-cache-username:
        description: "The username used to authenticate with the Gradle Build Cache Node."
        required: true
      gradle-cache-password:
        description: "The password used to authenticate with the Gradle Build Cache Node."
        required: true
    outputs:
      failure-mode:
        description: "Failure Mode"
        value: ${{ jobs.set-failure-mode.outputs.failure-mode }}

defaults:
  run:
    shell: bash

permissions:
  id-token: write
  contents: read

env:
  GRADLE_CACHE_USERNAME: ${{ secrets.gradle-cache-username }}
  GRADLE_CACHE_PASSWORD: ${{ secrets.gradle-cache-password }}
  DOCKER_MANIFEST_GENERATOR: .github/workflows/support/scripts/generate-docker-artifact-baseline.sh
  DOCKER_MANIFEST_PATH: ${{ github.workspace }}/.manifests/docker
  DOCKER_REGISTRY: localhost:5000
  DOCKER_IMAGE_NAME: consensus-node
  DOCKER_CONTEXT_PATH: hedera-node/infrastructure/docker/containers/production-next/consensus-node
  SKOPEO_VERSION: v1.14.0

jobs:
  generate-baseline:
    name: "Docker: Generate Baseline"
    runs-on: hl-cn-docker-determinism-lin-md
    outputs:
      sha: ${{ steps.commit.outputs.sha }}
      sha-abbrev: ${{ steps.commit.outputs.sha-abbrev }}
      source-date: ${{ steps.commit.outputs.source-date }}
      path: ${{ steps.baseline.outputs.path }}
      file: ${{ steps.baseline.outputs.file }}
      name: ${{ steps.baseline.outputs.name }}
      failure-mode: ${{ steps.set-failure-mode.outputs.failure-mode }}

    steps:
      - name: Prepare Job
        uses: pandaswhocode/initialize-github-job@a57cd6d8d768b2f3c95334bdd4fa8c21609fc651 # v1.0.5
        with:
          checkout: "true"
          checkout-ref: "${{ inputs.ref }}"
          checkout-token: "${{ secrets.github-token }}"
          checkout-fetch-depth: "1"
          setup-java: "true"
          java-distribution: "${{ inputs.java-distribution }}"
          java-version: "${{ inputs.java-version }}"
          setup-gradle: "true"
          gradle-cache-read-only: "false"

      - name: Authenticate to Google Cloud
        id: google-auth
        uses: step-security/google-github-auth@40f6deebd366f16c782d7a0ad0844e3b96a032a6 # v2.1.10
        with:
          workload_identity_provider: "projects/235822363393/locations/global/workloadIdentityPools/hedera-builds-pool/providers/hedera-builds-gh-actions"
          service_account: "swirlds-automation@hedera-registry.iam.gserviceaccount.com"

      - name: Setup Google Cloud SDK
        uses: google-github-actions/setup-gcloud@77e7a554d41e2ee56fc945c52dfd3f33d12def9a # v2.1.4

      - name: Retrieve Commit Hash
        id: commit
        run: |
          echo "sha=$(git rev-parse HEAD)" >> "${GITHUB_OUTPUT}"
          echo "sha-abbrev=$(git rev-parse HEAD | tr -d '[:space:]' | cut -c1-8)" >> "${GITHUB_OUTPUT}"
          echo "source-date=$(git log -1 --pretty=%ct)" >> "${GITHUB_OUTPUT}"

      - name: Baseline Existence Check
        id: baseline
        run: |
          BASELINE_NAME="${{ steps.commit.outputs.sha }}.tar.gz"
          BASELINE_PATH="gs://hedera-ci-ephemeral-artifacts/${{ github.repository }}/docker/baselines"
          BASELINE_FILE="${BASELINE_PATH}/${BASELINE_NAME}"
          BASELINE_EXISTS="false"

          if gsutil ls "${BASELINE_FILE}" >/dev/null 2>&1; then
             BASELINE_EXISTS="true"
          fi

          echo "exists=${BASELINE_EXISTS}" >> "${GITHUB_OUTPUT}"
          echo "path=${BASELINE_PATH}" >> "${GITHUB_OUTPUT}"
          echo "name=${BASELINE_NAME}" >> "${GITHUB_OUTPUT}"
          echo "file=${BASELINE_FILE}" >> "${GITHUB_OUTPUT}"

      - name: Install Skopeo and JQ
        if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }}
        run: |
          sudo apt-get update
          sudo apt-get install --yes --no-install-recommends skopeo jq

      - name: Setup QEmu Support
        uses: docker/setup-qemu-action@5964de0df58d5ad28b04d8fe2e6b80ad47105b91 # v3.5.0
        if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }}

      - name: Setup Docker Buildx Support
        uses: step-security/setup-buildx-action@f931205d68723ad9589fd2a7e2ece238bf9de341 # v4.0.0
        if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }}
        with:
          version: v0.32.1
          driver-opts: network=host,image=ghcr.io/hashgraph/runner-images/buildkit:buildx-stable-1
          buildkitd-config-inline: |
            [registry."docker.io"]
              mirrors = ["https://hub.mirror.docker.lat.ope.eng.hashgraph.io"]

      - name: Setup Local Docker Registry
        if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }}
        run: docker run -d -p 5000:5000 --restart=always --name registry registry:latest

      - name: Show Docker Version
        if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }}
        run: docker version

      - name: Show Docker Info
        if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }}
        run: docker info

      - name: Build Gradle Artifacts
        if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }}
        run: ./gradlew assemble

      - name: Prepare for Docker Build
        if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }}
        run: |
          mkdir -p "${{ github.workspace }}/${{ env.DOCKER_CONTEXT_PATH }}/sdk/data"

          echo "::group::Copying Library Artifacts"
            cp -Rvf "${{ github.workspace }}/hedera-node/data/lib" "${{ github.workspace }}/${{ env.DOCKER_CONTEXT_PATH }}/sdk/data/"
          echo "::endgroup::"

          echo "::group::Copying Application Artifacts"
            cp -Rvf "${{ github.workspace }}/hedera-node/data/apps" "${{ github.workspace }}/${{ env.DOCKER_CONTEXT_PATH }}/sdk/data/"
          echo "::endgroup::"

      - name: Write Artifact Version Descriptor
        if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }}
        run: |
          printf "VERSION=%s\nCOMMIT=%s\nDATE=%s" "$(./gradlew -q showVersion)" "$(git log -1 --format='%H' | cut -c1-8)" "$(date -u)" | tee "${{ github.workspace }}/${{ env.DOCKER_CONTEXT_PATH }}/sdk/VERSION"

      - name: Build Docker Image
        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
        env:
          SOURCE_DATE_EPOCH: ${{ steps.commit.outputs.source-date }}
        if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }}
        with:
          push: true
          no-cache: true
          platforms: linux/amd64,linux/arm64
          build-args: |
            SOURCE_DATE_EPOCH=${{ steps.commit.outputs.source-date }}
          context: ${{ env.DOCKER_CONTEXT_PATH }}
          tags: ${{ env.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE_NAME }}:${{ steps.commit.outputs.sha-abbrev }}

      - name: Generate Manifest
        id: manifest
        env:
          MANIFEST_PATH: ${{ env.DOCKER_MANIFEST_PATH }}
        if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }}
        run: GITHUB_SHA="${{ needs.generate-baseline.outputs.sha-abbrev }}" ${{ env.DOCKER_MANIFEST_GENERATOR }}

      - name: Amend Manifest with Gradle Artifacts
        if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }}
        working-directory: ${{ env.DOCKER_MANIFEST_PATH }}
        run: |
          EXTRACTED_FILE_NAME="${{ steps.commit.outputs.sha }}.tar"
          gunzip "${{ steps.manifest.outputs.name }}"
          tar -rvf "${EXTRACTED_FILE_NAME}" -C "${{ github.workspace }}/${{ env.DOCKER_CONTEXT_PATH }}" sdk
          gzip "${EXTRACTED_FILE_NAME}"

      - name: Upload Baseline
        if: ${{ steps.baseline.outputs.exists == 'false' && !failure() && !cancelled() }}
        run: gsutil cp "${{ steps.manifest.outputs.file }}" "${{ steps.baseline.outputs.file }}"

      - name: Set Failure Mode Output
        id: set-failure-mode
        if: ${{ always() }}
        run: |
          if ${{ success() }}; then
            echo "failure-mode=none" >> "${GITHUB_OUTPUT}"
          else
            echo "failure-mode=workflow" >> "${GITHUB_OUTPUT}"
          fi

  verify-artifacts:
    name: "Docker: Verify Artifacts (${{ join(matrix.os, ', ') }})"
    runs-on: ${{ matrix.os }}
    needs:
      - generate-baseline
    strategy:
      fail-fast: false
      matrix:
        # Windows is not supported due to GitHub not supporting Docker Desktop/Podman Desktop and Docker CE on Windows
        # not supporting BuildKit and the Buildx plugin.
        # GitHub hosted MacOS and Ubuntu runners are temporarily disabled.
        os:
          - ubuntu-24.04
          #- ubuntu-22.04
          #- macos-12
          #- macos-11
          #- hl-cn-docker-determinism-lin-md
          - hl-cn-docker-determinism-lin-lg
          - hl-cn-docker-determinism-lin-u24-lg
          - hl-cn-docker-determinism-lin-d12-lg
    outputs:
      failure-mode: ${{ steps.set-failure-mode.outputs.failure-mode }}
    steps:
      - name: Standardize Git Line Endings
        run: |
          git config --global core.autocrlf false
          git config --global core.eol lf

      - name: Prepare Job
        uses: pandaswhocode/initialize-github-job@a57cd6d8d768b2f3c95334bdd4fa8c21609fc651 # v1.0.5
        with:
          checkout: "true"
          checkout-ref: "${{ inputs.ref }}"
          checkout-token: "${{ secrets.github-token }}"
          checkout-fetch-depth: "1"
          setup-java: "true"
          java-distribution: "${{ inputs.java-distribution }}"
          java-version: "${{ inputs.java-version }}"
          setup-gradle: "true"
          gradle-cache-read-only: "false"

      - name: Install Python (Linux)
        if: ${{ runner.os == 'Linux' }}
        run: |
          sudo apt-get update
          sudo apt-get install --yes --no-install-recommends python3 python3-venv python3-pip

      - name: Install Python
        if: ${{ runner.os != 'Linux' }}
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "3.12"

      - name: Install JQ (Linux)
        if: ${{ runner.os == 'Linux' }}
        run: |
          sudo apt-get update
          sudo apt-get install --yes --no-install-recommends jq

      - name: Install Skopeo (Linux)
        if: ${{ runner.os == 'Linux' }}
        run: |
          source /etc/os-release
          if [[ "${VERSION_ID}" != "20.04" ]]; then
            sudo apt-get install --yes --no-install-recommends skopeo
          fi

      - name: Install Skopeo and JQ (macOS)
        if: ${{ runner.os == 'macOS' }}
        run: brew install skopeo jq

      - name: Setup CoreUtils (macOS)
        if: ${{ runner.os == 'macOS' }}
        run: brew install coreutils

      - name: Authenticate to Google Cloud
        id: google-auth
        uses: step-security/google-github-auth@40f6deebd366f16c782d7a0ad0844e3b96a032a6 # v2.1.10
        with:
          workload_identity_provider: "projects/235822363393/locations/global/workloadIdentityPools/hedera-builds-pool/providers/hedera-builds-gh-actions"
          service_account: "swirlds-automation@hedera-registry.iam.gserviceaccount.com"

      - name: Setup Google Cloud SDK
        uses: google-github-actions/setup-gcloud@77e7a554d41e2ee56fc945c52dfd3f33d12def9a # v2.1.4
        env:
          CLOUDSDK_PYTHON: ${{ format('{0}{1}', env.pythonLocation, runner.os == 'Windows' && '\python.exe' || '/bin/python3') }}

      - name: Download Baseline
        env:
          CLOUDSDK_PYTHON: ${{ format('{0}{1}', env.pythonLocation, runner.os == 'Windows' && '\python.exe' || '/bin/python3') }}
        run: |
          mkdir -p "${DOCKER_MANIFEST_PATH}"
          cd "${DOCKER_MANIFEST_PATH}"
          gsutil cp "${{ needs.generate-baseline.outputs.file }}" .
          tar -xzf "${{ needs.generate-baseline.outputs.name }}"
          mv "sdk" "${{ github.workspace }}/${{ env.DOCKER_CONTEXT_PATH }}/"

      - name: Remove Preinstalled Docker
        if: ${{ runner.os == 'macOS' }}
        run: |
          set -x
          sudo killall dockerd || true
          sudo killall containerd || true
          sudo rm -rvf /usr/bin/*containerd* || true
          sudo rm -rvf /usr/bin/docker* || true
          sudo rm -rvf /usr/local/bin/docker* || true
          sudo rm -rvf /usr/local/bin/*lima* || true

      - name: Install Lima (macOS)
        if: ${{ runner.os == 'macOS' }}
        run: |
          VERSION="v0.20.0"
          curl -fsSL "https://github.com/lima-vm/lima/releases/download/${VERSION}/lima-${VERSION:1}-$(uname -s)-$(uname -m).tar.gz" | sudo tar Cxzvm /usr/local

      - name: Determine Home Directory
        id: home
        run: echo "directory=$(tr -d '[:space:]' < <(cd ~ && pwd))" >> "${GITHUB_OUTPUT}"

      - name: Setup QEmu Support
        uses: docker/setup-qemu-action@5964de0df58d5ad28b04d8fe2e6b80ad47105b91 # v3.5.0

      - name: Setup Docker Buildx Support
        uses: step-security/setup-buildx-action@f931205d68723ad9589fd2a7e2ece238bf9de341 # v4.0.0
        with:
          version: v0.32.1
          driver-opts: network=host,image=ghcr.io/hashgraph/runner-images/buildkit:buildx-stable-1
          buildkitd-config-inline: |
            [registry."docker.io"]
              mirrors = ["https://hub.mirror.docker.lat.ope.eng.hashgraph.io"]

      - name: Setup Local Docker Registry
        run: docker run -d -p 5000:5000 --restart=always --name registry registry:latest

      - name: Show Docker Version
        run: docker version

      - name: Show Docker Info
        run: docker info

      - name: Build Docker Image
        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
        env:
          SOURCE_DATE_EPOCH: ${{ needs.generate-baseline.outputs.source-date }}
        with:
          push: true
          no-cache: true
          platforms: linux/amd64,linux/arm64
          build-args: |
            SOURCE_DATE_EPOCH=${{ needs.generate-baseline.outputs.source-date }}
          context: ${{ env.DOCKER_CONTEXT_PATH }}
          tags: ${{ env.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE_NAME }}:${{ needs.generate-baseline.outputs.sha-abbrev }}

      - name: Regenerate Manifest
        id: regen-manifest
        env:
          MANIFEST_PATH: ${{ env.DOCKER_MANIFEST_PATH }}/regenerated
        run: GITHUB_SHA="${{ needs.generate-baseline.outputs.sha-abbrev }}" ${{ env.DOCKER_MANIFEST_GENERATOR }}

      - name: Validate Layers (linux/amd64)
        run: |
          if ! diff -u "${DOCKER_MANIFEST_PATH}/linux-amd64.layers.json" "${{ steps.regen-manifest.outputs.path }}/linux-amd64.layers.json" >/dev/null 2>&1; then
           echo "::group::Layer Differences"
           diff -u "${DOCKER_MANIFEST_PATH}/linux-amd64.layers.json" "${{ steps.regen-manifest.outputs.path }}/linux-amd64.layers.json"
           echo "::endgroup::"
           exit 1
          fi

      - name: Validate Layers (linux/arm64)
        run: |
          if ! diff -u "${DOCKER_MANIFEST_PATH}/linux-arm64.layers.json" "${{ steps.regen-manifest.outputs.path }}/linux-arm64.layers.json" >/dev/null 2>&1; then
           echo "::group::Layer Differences"
           diff -u "${DOCKER_MANIFEST_PATH}/linux-arm64.layers.json" "${{ steps.regen-manifest.outputs.path }}/linux-arm64.layers.json"
           echo "::endgroup::"
           exit 1
          fi

      - name: Validate Full Manifest (linux/amd64)
        run: |
          if ! diff -u "${DOCKER_MANIFEST_PATH}/linux-amd64.comparable.json" "${{ steps.regen-manifest.outputs.path }}/linux-amd64.comparable.json" >/dev/null 2>&1; then
           echo "::group::Layer Differences"
           diff -u "${DOCKER_MANIFEST_PATH}/linux-amd64.comparable.json" "${{ steps.regen-manifest.outputs.path }}/linux-amd64.comparable.json"
           echo "::endgroup::"
           exit 1
          fi

      - name: Validate Full Manifest (linux/arm64)
        run: |
          if ! diff -u "${DOCKER_MANIFEST_PATH}/linux-arm64.comparable.json" "${{ steps.regen-manifest.outputs.path }}/linux-arm64.comparable.json" >/dev/null 2>&1; then
           echo "::group::Layer Differences"
           diff -u "${DOCKER_MANIFEST_PATH}/linux-arm64.comparable.json" "${{ steps.regen-manifest.outputs.path }}/linux-arm64.comparable.json"
           echo "::endgroup::"
           exit 1
          fi

      - name: Publish Manifests
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        if: ${{ steps.regen-manifest.conclusion == 'success' && failure() && !cancelled() }}
        with:
          name: Docker Manifests [${{ join(matrix.os, ', ') }}]
          path: ${{ env.DOCKER_MANIFEST_PATH }}/**

      - name: Set Failure Mode Output
        id: set-failure-mode
        if: ${{ always() }}
        run: |
          # If the job overall succeeded, mark as 'none', otherwise 'workflow'.
          if ${{ success() }}; then
            echo "failure-mode=none" >> "${GITHUB_OUTPUT}"
          else
            echo "failure-mode=workflow" >> "${GITHUB_OUTPUT}"
          fi

  set-failure-mode:
    name: "Set Failure Mode"
    runs-on: hl-cn-default-lin-sm
    needs:
      - generate-baseline
      - verify-artifacts
    if: ${{ !cancelled() && always() }}
    outputs:
      failure-mode: ${{ steps.set-failure-mode.outputs.failure-mode }}
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
        with:
          egress-policy: audit

      - name: Set Failure Mode
        id: set-failure-mode
        run: |
          job_statuses=(
            "${{ needs.generate-baseline.result || 'skipped' }}"
            "${{ needs.verify-artifacts.result || 'skipped' }}"
          )

          job_failure_modes=(
            "${{ needs.generate-baseline.outputs.failure-mode || 'none' }}"
            "${{ needs.verify-artifacts.outputs.failure-mode || 'none' }}"
          )

          failure_mode="none"

          for i in "${!job_statuses[@]}"; do
            if [[ "${job_statuses[$i]}" == "failure" ]]; then
              mode="${job_failure_modes[$i]}"
              if [[ "${failure_mode}" == "none" ]]; then
                failure_mode="${mode}"
              elif [[ "${mode}" == "workflow" ]]; then
                failure_mode="workflow"
              fi
            fi
          done

          echo "failure-mode=${failure_mode}" >> "${GITHUB_OUTPUT}"