scheduled.yml

name: Scheduled

on:
  schedule:
    - cron:  '45 * * * *'
  workflow_dispatch:

concurrency:
  group: "scheduled"
  cancel-in-progress: true

jobs:
  check:
    runs-on: ubuntu-latest
    outputs:
      configs: ${{ steps.repo_check.outputs.configs }}
    steps:
      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
      - uses: mikefarah/yq@065b200af9851db0d5132f50bc10b1406ea5c0a8 # v4.50.1
      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: '3.10'
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install pyyaml
      - name: Generate config.yaml
        run: |
          python generate_config.py
          echo "Generated config.yaml for workflow use"
      - name: Login to Docker Hub
        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
        with:
          username: ${{ vars.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
      - id: repo_check
        env:
          CONCURRENT_IMAGE_PULL: ${{ vars.CONCURRENT_IMAGE_PULL }}
        run: |
          # This script reads config.yaml, platforms.yaml and runners.yaml to generate a list of configurations to build and deploy
          # It will:
          #    - check the source respository for the latest commit hash
          #    - check if the built image exists in our dockerhub registry
          #    - generate a list of configurations to build and deploy

          CONFIG_FILE="config.yaml"
          PLATFORMS_FILE="platforms.yaml"
          RUNNERS_FILE="runners.yaml"

          # Create a temporary directory for storing intermediate results
          TEMP_DIR=$(mktemp -d)
          # Ensure the temporary directory is removed when the script exits
          trap "rm -rf $TEMP_DIR" EXIT

          process_commits() {
              local LINE=$1
              local SOURCE_REPOSITORY=$2
              local SOURCE_REF=$3
              local TARGET_REPOSITORY=$4
              local TARGET_TAG=$5

              local CLIENT="${TARGET_REPOSITORY#*/}"
              local RESPONSE=$(curl -s -H "Accept: application/vnd.github+json" \
                  -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
                  "https://api.github.com/repos/${SOURCE_REPOSITORY}/commits/${SOURCE_REF}?per_page=1")
              local COMMIT_HASH_FULL=$(echo "$RESPONSE" | jq -r '.sha')
              local COMMIT_HASH=$(echo "$COMMIT_HASH_FULL" | cut -c1-7)

              if [[ -z "$COMMIT_HASH" || "$COMMIT_HASH" == "null" ]]; then
                  # Log error but don't exit; just skip this configuration
                  echo "[LINE:$LINE] Error fetching commit hash for ${SOURCE_REPOSITORY}#${SOURCE_REF}, skipping."
                  return
              fi

              local configOutput="${TEMP_DIR}/${LINE}_commits.json"
              touch $configOutput

              echo "{\"line\": \"$LINE\", \"commit_hash\": \"$COMMIT_HASH\", \"commit_hash_full\": \"$COMMIT_HASH_FULL\"}," >> $configOutput
          }

          process_image() {
              local LINE=$1
              local IMAGE=$2
              local URL=$3

              local imageOutput="${TEMP_DIR}/${LINE}_image.json"
              touch $imageOutput
              local exists=$(curl -s $URL | jq '.results | length > 0')

              # check if exists == true
              if [ "$exists" == "true" ]; then
                exists=true
              else
                exists=false
              fi

              echo "{\"line\": \"$LINE\", \"image\": \"$IMAGE\", \"exists\": $exists}" >> $imageOutput
          }

          # Get commit hashes for each configuration in parallel
          while IFS=$'\t' read -r LINE SOURCE_REPOSITORY SOURCE_REF TARGET_REPOSITORY TARGET_TAG; do
              process_commits "$LINE" "$SOURCE_REPOSITORY" "$SOURCE_REF" "$TARGET_REPOSITORY" "$TARGET_TAG" &
          done < <(yq -r 'to_entries | map_values({"value":.value, "index":.key}) | .[] | [.index, .value.source.repository, .value.source.ref, .value.target.repository, .value.target.tag] | @tsv' "$CONFIG_FILE")

          wait

          # Initialize JSON arrays
          COMMITS="["

          # Concatenate results, ensuring files exist before attempting to read
          for file in $TEMP_DIR/*_commits.json; do
              if [ -f "$file" ]; then
                  COMMITS+=$(cat "$file")
              fi
          done

          # Remove trailing commas and close JSON arrays
          COMMITS="${COMMITS%,}]"

          echo "Checking if images exist in dockerhub..."
          while IFS=$'\t' read -r LINE SOURCE_REPOSITORY SOURCE_REF TARGET_REPOSITORY TARGET_TAG; do
              # get the image commit hash from LINE
              COMMIT_HASH=$(echo "$COMMITS" | jq -r --arg LINE "$LINE" '.[] | select(.line == $LINE) | .commit_hash')
              IMAGE_TAG="${TARGET_TAG}-${COMMIT_HASH}"
              IMAGE="${TARGET_REPOSITORY}:${IMAGE_TAG}"
              URL="https://hub.docker.com/v2/repositories/${TARGET_REPOSITORY}/tags?page_size=25&page=1&ordering=&name=${IMAGE_TAG}"
              process_image $LINE $IMAGE $URL &
          done < <(yq -r 'to_entries | map_values({"value":.value, "index":.key}) | .[] | [.index, .value.source.repository, .value.source.ref, .value.target.repository, .value.target.tag] | @tsv' "$CONFIG_FILE")

          wait

          declare -A images

          # Concatenate results, ensuring files exist before attempting to read
          for file in $TEMP_DIR/*_image.json; do
              if [ -f "$file" ]; then
                  LINE=$(cat "$file" | jq -r '.line')
                  IMAGE=$(cat "$file" | jq -r '.image')
                  EXISTS=$(cat "$file" | jq -r '.exists')
                  images[$IMAGE]=$EXISTS
              fi
          done

          CONFIGS="configs=["

          echo "Generating configuration files..."
          while IFS=$'\t' read -r LINE SOURCE_REPOSITORY SOURCE_REF SOURCE_PATCH TARGET_REPOSITORY TARGET_TAG; do
              # get the image commit hash from LINE
              COMMIT_HASH=$(echo "$COMMITS" | jq -r --arg LINE "$LINE" '.[] | select(.line == $LINE) | .commit_hash')
              COMMIT_HASH_FULL=$(echo "$COMMITS" | jq -r --arg LINE "$LINE" '.[] | select(.line == $LINE) | .commit_hash_full')
              IMAGE_TAG="${TARGET_TAG}-${COMMIT_HASH}"
              IMAGE="${TARGET_REPOSITORY}:${IMAGE_TAG}"
              CLIENT="${TARGET_REPOSITORY#*/}"

              # Build if image doesn't exist OR if we couldn't determine existence (fail-safe)
              if [ -z "${images[$IMAGE]}" ] || [ "${images[$IMAGE]}" == "false" ]; then
                # Handle platforms and runners, ensuring output files are created even if empty
                platforms=$(yq e ".$CLIENT[]" "$PLATFORMS_FILE")
                platformsArr=""

                for platform in $platforms; do
                  runner=$(yq e ".\"$platform\"" "$RUNNERS_FILE")
                  slug=$(echo "$platform" | tr '/' '-')
                  platformsArr+="{\\\"platform\\\": \\\"$platform\\\", \\\"runner\\\": \\\"$runner\\\", \\\"slug\\\": \\\"$slug\\\"},"
                done
                platformsArr="${platformsArr%,}"
                # convert to string
                platformsOutput="{\"platforms\": \"[$platformsArr]\"}"

                # Convert NONE placeholder back to empty string for patch
                if [ "$SOURCE_PATCH" == "NONE" ]; then
                  SOURCE_PATCH=""
                fi

                CONFIGS+=$(echo "$(yq -r -o=json ".[${LINE}]" "$CONFIG_FILE" | jq --argjson plat "$platformsOutput" --arg commit "$COMMIT_HASH_FULL" --arg patch "$SOURCE_PATCH" '. + $plat + {source_commit: $commit, source_patch: $patch}'),")
              fi
          done < <(yq -r 'to_entries | map_values({"value":.value, "index":.key}) | .[] | [.index, .value.source.repository, .value.source.ref, .value.source.patch // "NONE", .value.target.repository, .value.target.tag] | @tsv' "$CONFIG_FILE")

          # Remove trailing commas and close JSON arrays
          CONFIGS="${CONFIGS%,}]"

          echo "CONFIGS: $CONFIGS"
          echo $CONFIGS >> $GITHUB_OUTPUT

  deploy:
    needs: check
    if: ${{ needs.check.outputs.configs != '[]' && needs.check.outputs.configs != '' }}
    uses: ./.github/workflows/deploy.yml
    strategy:
      fail-fast: false
      matrix:
        config: ${{fromJson(needs.check.outputs.configs)}}
    name: ${{ matrix.config.source.repository }}#${{ matrix.config.source.ref }} ${{ matrix.config.target.tag }}
    with:
      source_repository: ${{ matrix.config.source.repository }}
      source_ref: ${{ matrix.config.source.ref }}
      source_commit: ${{ matrix.config.source_commit }}
      source_patch: ${{ matrix.config.source_patch }}
      build_script: ${{ matrix.config.build_script }}
      build_args: "${{ matrix.config.build_args }}"
      target_tag: ${{ matrix.config.target.tag }}
      target_repository: ${{ matrix.config.target.repository }}
      target_dockerfile: ${{ matrix.config.target.dockerfile }}
      platforms: ${{ matrix.config.platforms }}
      harbor_registry: "${{ vars.HARBOR_REGISTRY }}"
      HARBOR_USERNAME: "${{ vars.HARBOR_USERNAME }}"
      DOCKER_USERNAME: "${{ vars.DOCKER_USERNAME }}"
      GOPROXY: "${{ vars.GOPROXY }}"
    secrets:
      DOCKER_PASSWORD: "${{ secrets.DOCKER_PASSWORD }}"
      HARBOR_PASSWORD: "${{ secrets.HARBOR_PASSWORD }}"
      MACOS_PASSWORD: "${{ secrets.MACOS_PASSWORD }}"

  notify:
    name: Discord Notification
    runs-on: ubuntu-latest
    needs:
      - check
      - deploy
    if: cancelled() || failure()
    steps:
      - name: Notify
        uses: nobrayner/discord-webhook@1766a33bf571acdcc0678f00da4fb83aad01ebc7 # v1
        with:
          github-token: ${{ secrets.github_token }}
          discord-webhook: ${{ secrets.DISCORD_WEBHOOK }}