|
| 1 | +# Action to build and deploy Docker images from the CI Dockerfiles in the |
| 2 | +# repo. |
| 3 | +# |
| 4 | +# The action is triggered by a push to the `main`/`master` or `testing` |
| 5 | +# branch when it affects one of the CI `Dockerfile`s |
| 6 | +# (`buildkite/docker/*/Dockerfile`). It can also be triggered manually via |
| 7 | +# the Actions web UI, the GH REST API or the GH CLI tool, e.g.: |
| 8 | +# ```sh |
| 9 | +# gh workflow run build-ci-docker-images |
| 10 | +# ``` |
| 11 | +# |
| 12 | +# When triggered by a `push` event the action will: |
| 13 | +# |
| 14 | +# 1. Determine which `Dockerfile`s are affected. |
| 15 | +# |
| 16 | +# 2. For those `Dockerfile`s, determine its build context (the directory |
| 17 | +# where the `Dockerfile` lives) and the build targets in it. |
| 18 | +# |
| 19 | +# 3. Filter the build targets: |
| 20 | +# * first with `RE_TARGET_INCLUDE`, which defaults to empty so it will |
| 21 | +# match all build targets. |
| 22 | +# * then with `RE_TARGET_EXCLUDE`: set by default to remove some of the |
| 23 | +# build targets the deprecated images like `centos7` and some targets |
| 24 | +# that we don't want to build like the `nojdk` ones. |
| 25 | +# |
| 26 | +# 5. Then, it will also exclude all `testimage` targets because that image |
| 27 | +# is only used for manually testing the workflow. |
| 28 | +# |
| 29 | +# 6. Finally, it will spawn a `docker/build-push-action` job for each of |
| 30 | +# the build targets. For every image built it will push to the registry |
| 31 | +# three image tags: |
| 32 | +# * a `sha` tag with the short hash of the commit that triggered the push |
| 33 | +# * a `date` tag with the current date in ISO format (`YYYYMMDD`) |
| 34 | +# * a `latest` tag |
| 35 | +# |
| 36 | +# When triggered manually (`workflow_dispatch` event) the workflow will |
| 37 | +# default to "running in test mode": it will follow the same steps as a |
| 38 | +# `push` run but with different default values (see |
| 39 | +# `workflow_dispatch.inputs`): |
| 40 | +# * `RE_TARGET_INCLUDE` set to `testimage` (a very simple image to |
| 41 | +# exercise the build that doesn't take much compute) and |
| 42 | +# * `RE_TARGET_EXCLUDE` set to the same pattern as in the `push` event. |
| 43 | +# |
| 44 | +# This effectively limits the build targets to only those in `testimage` not |
| 45 | +# excluded by `RE_TARGET_EXCLUDE` (e.g. `nojdk`). |
| 46 | +# |
| 47 | +# The "test run" also limits the `PLATFORMS` to `linux/amd64`, to further |
| 48 | +# reduce the cost and time of a test run. |
| 49 | +# |
| 50 | +# Finally, it will build those `testimage` targets but **it won't tag |
| 51 | +# `latest` or push any of the image tags to the registry**. |
| 52 | +# |
| 53 | +# This "test mode" behavior can be changed by setting the |
| 54 | +# `workflow_dispatch.inputs` variables: `RE_TARGET_EXCLUDE`, |
| 55 | +# `RE_TARGET_INCLUDE`, `PLATFORMS`, `TAG_DATE`, `TAG_LATEST` and `PUSH`, |
| 56 | +# e.g.: |
| 57 | +# ```sh |
| 58 | +# gh workflow run build-ci-docker-images \ |
| 59 | +# -f RE_TARGET_INCLUDE=ubuntu2404 -f TAG_DATE=20241101 |
| 60 | +# ``` |
| 61 | + |
| 62 | +name: build-ci-docker-images |
| 63 | + |
| 64 | +on: |
| 65 | + push: |
| 66 | + branches: |
| 67 | + - main |
| 68 | + - master |
| 69 | + - testing |
| 70 | + paths: |
| 71 | + # NOTE: keep in-sync with env.SPARSE_PATH |
| 72 | + - buildkite/docker/*/Dockerfile |
| 73 | + |
| 74 | + workflow_dispatch: |
| 75 | + inputs: |
| 76 | + RE_TARGET_EXCLUDE: |
| 77 | + description: |- |
| 78 | + Filter out Docker targets matching this extended regex pattern |
| 79 | + # NOTE: keep in-sync with env.RE_TARGET_EXCLUDE |
| 80 | + default: ^centos7|^ubuntu16|nojdk |
| 81 | + RE_TARGET_INCLUDE: |
| 82 | + description: |- |
| 83 | + Select Docker targets matching this extended regex pattern |
| 84 | + # NOTE: keep in-sync with env.TEST_IMAGE |
| 85 | + default: testimage |
| 86 | + PLATFORMS: |
| 87 | + default: linux/amd64 |
| 88 | + TAG_DATE: |
| 89 | + description: Tag image date in ISO format (YYYYMMDD) |
| 90 | + TAG_LATEST: |
| 91 | + description: Tag image version as 'latest' |
| 92 | + default: false |
| 93 | + PUSH: |
| 94 | + description: Push images to registry |
| 95 | + default: false |
| 96 | + |
| 97 | +env: |
| 98 | + REGISTRY: ghcr.io |
| 99 | + SPARSE_PATH: buildkite/docker |
| 100 | + TEST_IMAGE: testimage |
| 101 | + |
| 102 | + GH_EVENT_NAME: ${{ github.event_name }} |
| 103 | + GH_REF_NAME: ${{ github.ref_name }} |
| 104 | + GH_REPO: ${{ github.repository }} |
| 105 | + GH_SHA: ${{ github.sha }} |
| 106 | + GH_SHA_BEFORE: ${{ github.event.before }} |
| 107 | + |
| 108 | + RE_TARGET_EXCLUDE: ${{ inputs.RE_TARGET_EXCLUDE || '^centos7|^ubuntu16|nojdk' }} |
| 109 | + RE_TARGET_INCLUDE: ${{ inputs.RE_TARGET_INCLUDE }} |
| 110 | + PLATFORMS: ${{ inputs.PLATFORMS || 'linux/amd64,linux/arm64' }} |
| 111 | + TAG_DATE: ${{ inputs.TAG_DATE }} |
| 112 | + TAG_LATEST: ${{ inputs.TAG_LATEST }} |
| 113 | + PUSH: ${{ github.event_name == 'push' || inputs.PUSH }} |
| 114 | + |
| 115 | +jobs: |
| 116 | + setup-targets: |
| 117 | + runs-on: ubuntu-latest |
| 118 | + |
| 119 | + defaults: |
| 120 | + run: |
| 121 | + shell: bash --noprofile --norc -euo pipefail {0} |
| 122 | + |
| 123 | + outputs: |
| 124 | + targets: ${{ steps.define_targets.outputs.targets }} |
| 125 | + targets_length: ${{ steps.define_targets.outputs.targets_length }} |
| 126 | + |
| 127 | + steps: |
| 128 | + - name: Sparse checkout SPARSE_PATH |
| 129 | + uses: actions/checkout@v4 |
| 130 | + with: |
| 131 | + sparse-checkout-cone-mode: false |
| 132 | + sparse-checkout: |- |
| 133 | + ${{ env.SPARSE_PATH }} |
| 134 | +
|
| 135 | + - name: Get DOCKERFILES |
| 136 | + run: |- |
| 137 | + # NOTE: |
| 138 | + # GH_SHA_BEFORE is empty on pushing the first commit of a new branch |
| 139 | + # or when running manually via workflow_dispatch |
| 140 | + if [[ -z $GH_SHA_BEFORE ]]; then |
| 141 | + DOCKERFILES="$(ls $SPARSE_PATH/*/Dockerfile)" |
| 142 | + else |
| 143 | + DOCKERFILES="$( |
| 144 | + git diff --name-only $GH_SHA_BEFORE $GH_SHA | |
| 145 | + grep Dockerfile || { |
| 146 | + # NOTE: |
| 147 | + # this grep could fail if e.g. we are force-pushing a stack |
| 148 | + # of commits where one or more commits do change Dockerfiles |
| 149 | + # but there's no change to any Dockerfile between the last |
| 150 | + # commit and this forced push. |
| 151 | + echo "WARNING: EMPTY grep" >&2 |
| 152 | + true |
| 153 | + } |
| 154 | + )" |
| 155 | + fi |
| 156 | +
|
| 157 | + DOCKERFILES_JSON="$(echo -n "$DOCKERFILES" | jq -R '.' | jq -sc '.' )" |
| 158 | +
|
| 159 | + echo "DOCKERFILES_JSON=$DOCKERFILES_JSON" |
| 160 | + echo "DOCKERFILES_JSON=$DOCKERFILES_JSON" >> $GITHUB_ENV |
| 161 | +
|
| 162 | + - name: Define targets |
| 163 | + id: define_targets |
| 164 | + run: |- |
| 165 | + if [[ "$GH_EVENT_NAME" == "push" ]]; then |
| 166 | + RE_TARGET_EXCLUDE="$RE_TARGET_EXCLUDE|$TEST_IMAGE" |
| 167 | + fi |
| 168 | +
|
| 169 | + DOCKERFILES=($(echo "$DOCKERFILES_JSON" | jq -r 'join(" ")')) |
| 170 | +
|
| 171 | + if [[ ${#DOCKERFILES[@]} -gt 0 ]]; then |
| 172 | + TARGETS_LS="$( |
| 173 | + grep -i '^FROM .* AS ' ${DOCKERFILES[@]} | |
| 174 | + awk '{print $NF}' | |
| 175 | + { grep -E "$RE_TARGET_INCLUDE" || true; } | |
| 176 | + { grep -vE "$RE_TARGET_EXCLUDE" || true; } | |
| 177 | + jq -R '.' |
| 178 | + )" |
| 179 | + else |
| 180 | + TARGETS_LS="" |
| 181 | + fi |
| 182 | +
|
| 183 | + TARGETS="$(echo "$TARGETS_LS" | jq -sc '.')" |
| 184 | + TARGETS_LENGTH="$(echo "$TARGETS_LS" | jq -s 'length')" |
| 185 | +
|
| 186 | + echo "targets=$TARGETS" |
| 187 | + echo "targets_length=$TARGETS_LENGTH" |
| 188 | +
|
| 189 | + echo "targets=$TARGETS" >> "$GITHUB_OUTPUT" |
| 190 | + echo "targets_length=$TARGETS_LENGTH" >> "$GITHUB_OUTPUT" |
| 191 | +
|
| 192 | + build-and-publish-docker-images: |
| 193 | + needs: setup-targets |
| 194 | + if: ${{ needs.setup-targets.outputs.targets_length > 0 }} |
| 195 | + |
| 196 | + runs-on: ubuntu-latest |
| 197 | + |
| 198 | + strategy: |
| 199 | + matrix: |
| 200 | + target: ${{ fromJson(needs.setup-targets.outputs.targets) }} |
| 201 | + |
| 202 | + defaults: |
| 203 | + run: |
| 204 | + shell: bash --noprofile --norc -euo pipefail {0} |
| 205 | + |
| 206 | + steps: |
| 207 | + - name: Sparse checkout SPARSE_PATH |
| 208 | + uses: actions/checkout@v4 |
| 209 | + with: |
| 210 | + sparse-checkout-cone-mode: false |
| 211 | + sparse-checkout: |- |
| 212 | + ${{ env.SPARSE_PATH }} |
| 213 | +
|
| 214 | + - name: Set up dynamic env |
| 215 | + env: |
| 216 | + MATRIX_TARGET: ${{ matrix.target }} |
| 217 | + run: |- |
| 218 | + declare -A TAGS |
| 219 | +
|
| 220 | + TAGS[sha]="${GH_SHA::7}" |
| 221 | +
|
| 222 | + TAGS[date]="$TAG_DATE" |
| 223 | + # set default date value if TAG_DATE is not set, is empty |
| 224 | + # or is an empty string |
| 225 | + if [[ -z "${TAGS[date]+isset}" || -z "${TAGS[date]// }" ]]; then |
| 226 | + TAGS[date]="$(date +%Y%m%d)" |
| 227 | + fi |
| 228 | +
|
| 229 | + if [[ "$GH_EVENT_NAME" == "push" || "$TAG_LATEST" == "true" ]]; then |
| 230 | + TAGS[latest]="latest" |
| 231 | + fi |
| 232 | +
|
| 233 | + if [[ "$REGISTRY" == "gcr.io" ]]; then |
| 234 | + IMAGE_PREFIX="bazel-public" |
| 235 | + elif [[ "$REGISTRY" == "ghcr.io" ]]; then |
| 236 | + IMAGE_PREFIX="$GH_REPO" |
| 237 | + else |
| 238 | + echo "Invalid registry: $REGISTRY" |
| 239 | + exit 1 |
| 240 | + fi |
| 241 | +
|
| 242 | + if [[ "$GH_REF_NAME" == "testing" ]]; then |
| 243 | + IMAGE_PREFIX="$IMAGE_PREFIX/testing" |
| 244 | + fi |
| 245 | +
|
| 246 | + IMAGE_NAME="$REGISTRY/$IMAGE_PREFIX/$MATRIX_TARGET" |
| 247 | +
|
| 248 | + DISTRO="${MATRIX_TARGET%%-*}" |
| 249 | + CONTEXT="$SPARSE_PATH/$DISTRO" |
| 250 | +
|
| 251 | + { |
| 252 | + for tag in ${!TAGS[@]}; do |
| 253 | + echo "IMAGE_TAG_${tag^^}=$IMAGE_NAME:${TAGS[$tag]}" |
| 254 | + done |
| 255 | +
|
| 256 | + echo "CONTEXT=$CONTEXT" |
| 257 | + } >> $GITHUB_ENV |
| 258 | +
|
| 259 | + - name: Set up QEMU |
| 260 | + uses: docker/setup-qemu-action@v3 |
| 261 | + |
| 262 | + - name: Set up Docker Buildx |
| 263 | + uses: docker/setup-buildx-action@v3 |
| 264 | + |
| 265 | + - name: Login to Registry |
| 266 | + uses: docker/login-action@v3 |
| 267 | + with: |
| 268 | + registry: ${{ env.REGISTRY }} |
| 269 | + username: ${{ github.actor }} |
| 270 | + password: ${{ secrets.GITHUB_TOKEN }} |
| 271 | + |
| 272 | + - name: Build and push |
| 273 | + uses: docker/build-push-action@v6 |
| 274 | + with: |
| 275 | + context: ${{ env.CONTEXT }} |
| 276 | + target: ${{ matrix.target }} |
| 277 | + platforms: ${{ env.PLATFORMS }} |
| 278 | + tags: |- |
| 279 | + ${{ env.IMAGE_TAG_SHA }} |
| 280 | + ${{ env.IMAGE_TAG_DATE }} |
| 281 | + ${{ env.IMAGE_TAG_LATEST }} |
| 282 | + labels: |- |
| 283 | + org.opencontainers.image.source=${{ github.repositoryUrl }} |
| 284 | + push: ${{ env.PUSH }} |
0 commit comments