feat: optimize skip_push and skip_retag #3
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build docker artifacts | ||
|
Check failure on line 1 in .github/workflows/build-docker-artifacts.yml
|
||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| builds: | ||
| type: string | ||
| required: false | ||
| default: "" | ||
| description: "Comma separated list of build keys. Leave empty to run for all." | ||
| push_to: | ||
| type: string | ||
| required: false | ||
| default: "" | ||
| description: "Comma separated list of push keys. Leave empty to trigger for all." | ||
| branch: | ||
| type: string | ||
| required: false | ||
| skip_push: | ||
| description: "Skip retagging and pushing the built images to customer registries" | ||
| type: boolean | ||
| required: false | ||
| default: false | ||
| fail_fast: | ||
| type: boolean | ||
| required: false | ||
| default: true | ||
| scan_high_severity: | ||
| description: "Include high severity" | ||
| type: boolean | ||
| required: false | ||
| default: true | ||
| runs_on: | ||
| type: string | ||
| required: false | ||
| default: "ubuntu-22.04" | ||
| secrets: | ||
| DATAVISYN_BOT_REPO_TOKEN: | ||
| required: false | ||
| CHECKOUT_TOKEN: | ||
| required: false | ||
| description: "Token to use for the checkout actions to access private repositories" | ||
| concurrency: | ||
| group: "${{ github.workflow }}-${{ github.ref || github.head_ref }}-${{ inputs.branch }}-${{ inputs.builds }}-${{ inputs.push_to }}" | ||
| cancel-in-progress: true | ||
| env: | ||
| WORKFLOW_BRANCH: "main" | ||
| DATAVISYN_PYTHON_BASE_IMAGE: "188237246440.dkr.ecr.eu-central-1.amazonaws.com/datavisyn/base/python:main" | ||
| DATAVISYN_NGINX_BASE_IMAGE: "188237246440.dkr.ecr.eu-central-1.amazonaws.com/datavisyn/base/nginx:main" | ||
| permissions: | ||
| contents: read | ||
| id-token: write | ||
| jobs: | ||
| get-flavors: | ||
| name: Get flavors from config.json | ||
| outputs: | ||
| result: ${{ steps.get-flavors.outputs.result }} | ||
| # Do not run this on self-hosted, as it is faster and shouldn't be blocking anything | ||
| # runs-on: ${{ inputs.runs_on || 'ubuntu-22.04' }} | ||
| runs-on: "ubuntu-22.04" | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v5 | ||
| with: | ||
| ref: ${{ inputs.branch || github.sha }} | ||
| token: ${{ secrets.CHECKOUT_TOKEN || github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} | ||
| - name: Checkout github-workflows repository | ||
| uses: actions/checkout@v5 | ||
| with: | ||
| repository: datavisyn/github-workflows | ||
| ref: ${{ env.WORKFLOW_BRANCH }} | ||
| path: ./tmp/github-workflows | ||
| - name: Validate ./deploy/build/config.json | ||
| shell: bash | ||
| run: | | ||
| # Validate the config with the schema | ||
| python -m venv .venv | ||
| source .venv/bin/activate | ||
| pip install jsonschema | ||
| jsonschema -i ./deploy/build/config.json ./tmp/github-workflows/.github/workflows/build-docker-artifacts-config.schema.json | ||
| deactivate | ||
| rm -rf .venv | ||
| - name: Get all flavors and components from ./deploy/build/config.json | ||
| uses: actions/github-script@v7 | ||
| id: get-flavors | ||
| with: | ||
| script: | | ||
| const fs = require('fs'); | ||
| const path = require('path'); | ||
| const config = require('./deploy/build/config.json'); | ||
| const buildTime = new Date().toISOString().replace(/:/g, '').replace(/\..+/, 'Z'); | ||
| const imageTagBranchName = "${{ github.ref }}".replace('refs/heads/', '').replace('refs/tags/', '').replace(/[^a-zA-Z0-9._-]/g, '-'); | ||
| const imageTag = `tagged-${imageTagBranchName}-${buildTime}`; | ||
| const builds = process.env.BUILDS ? process.env.BUILDS.split(',') : Object.keys(config.build); | ||
| const push_to = process.env.PUSH_TO ? process.env.PUSH_TO.split(',') : Object.keys(config.push || {}); | ||
| const skip_push = process.env.SKIP_PUSH === 'true'; | ||
| const flavors = builds.map(id => [id, config.build[id]]).filter(([id, flavor]) => flavor.skip !== true).map(([id, flavor]) => { | ||
| const hooksDirectory = path.join('./deploy/build', flavor.directory, 'hooks'); | ||
| const testImagesHookScript = path.join(hooksDirectory, 'test-images.sh'); | ||
| const testImagesHookReport = path.join(hooksDirectory, 'test-images-report'); | ||
| const testImageEnabled = fs.existsSync(testImagesHookScript); | ||
| return { | ||
| ...flavor, | ||
| id, | ||
| // We can skip the push only if we don't want to test the image below | ||
| skip_push: skip_push && !testImageEnabled, | ||
| skip_retag: skip_push, | ||
| // Add metadata to the flavor object (will be used as matrix input) | ||
| build_time: buildTime, | ||
| image_tag: imageTag, | ||
| image_tag_branch_name: imageTagBranchName, | ||
| ecr_respositories: flavor.components.map(component => component.ecr_repository), | ||
| test_images: { | ||
| enabled: testImageEnabled, | ||
| script_path: testImagesHookScript, | ||
| report_path: testImagesHookReport, | ||
| }, | ||
| components: flavor.components.map(component => { | ||
| // Format build arguments as build-args string | ||
| const formattedBuildArgs = component.build_args ? | ||
| Object.entries(component.build_args).map(([key, value]) => `${key}=${value}`).join('\n') : ''; | ||
| return { | ||
| ...component, | ||
| // We can skip the push only if we don't want to test the image below | ||
| skip_push: skip_push && !testImageEnabled, | ||
| skip_retag: skip_push, | ||
| // Add metadata to the component object (will be used as matrix input), | ||
| flavor, | ||
| flavor_id: id, | ||
| flavor_directory: `./deploy/build/${flavor.directory}`, | ||
| build_time: buildTime, | ||
| image_tag: imageTag, | ||
| image_ref: `${{ vars.DV_AWS_ECR_REGISTRY }}/${component.ecr_repository}:${imageTag}`, | ||
| image_tag_branch_name: imageTagBranchName, | ||
| formatted_build_args: formattedBuildArgs, | ||
| }; | ||
| }), | ||
| }; | ||
| }); | ||
| const flattenedComponents = flavors.flatMap(flavor => flavor.components); | ||
| const result = { | ||
| flavors, | ||
| components: flattenedComponents, | ||
| push_to: push_to.join(','), | ||
| }; | ||
| console.log(result); | ||
| return result; | ||
| env: | ||
| SKIP_PUSH: ${{ inputs.skip_push }} | ||
| BUILDS: ${{ inputs.builds }} | ||
| PUSH_TO: ${{ inputs.push_to }} | ||
| build-flavors: | ||
| name: Build ${{ matrix.component.directory }} of ${{ matrix.component.flavor.directory }} (${{ matrix.component.ecr_repository }}:${{ matrix.component.image_tag }}) | ||
| needs: get-flavors | ||
| strategy: | ||
| fail-fast: ${{ inputs.fail_fast }} | ||
| matrix: | ||
| component: ${{ fromJson(needs.get-flavors.outputs.result).components }} | ||
| runs-on: ${{ vars.BUILD_DOCKER_RUNS_ON_OVERRIDE || vars.RUNS_ON_OVERRIDE || inputs.runs_on || 'ubuntu-22.04' }} | ||
| steps: | ||
| - name: View flavor and component | ||
| shell: bash | ||
| run: | | ||
| echo "Component ${{ toJson(matrix.component) }}" | ||
| - name: Remove unnecessary files | ||
| if: ${{ vars.BUILD_DOCKER_NEEDS_MORE_SPACE == 'true' }} | ||
| run: | | ||
| sudo rm -rf /usr/share/dotnet | ||
| sudo rm -rf /usr/local/lib/android | ||
| sudo rm -rf /opt/ghc | ||
| # TODO: Support arbitrary repositories, not just the current one? | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v5 | ||
| with: | ||
| ref: ${{ inputs.branch || github.sha }} | ||
| token: ${{ secrets.CHECKOUT_TOKEN || github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} | ||
| # This is required such that yarn install can access private repositories, i.e. visyn_pro | ||
| # https://github.com/yarnpkg/yarn/issues/2614#issuecomment-2148174789 | ||
| persist-credentials: false | ||
| - name: Checkout github-workflows repository | ||
| uses: actions/checkout@v5 | ||
| with: | ||
| repository: datavisyn/github-workflows | ||
| ref: ${{ env.WORKFLOW_BRANCH }} | ||
| path: ./tmp/github-workflows | ||
| # This is required such that yarn install can access private repositories, i.e. visyn_pro | ||
| # https://github.com/yarnpkg/yarn/issues/2614#issuecomment-2148174789 | ||
| persist-credentials: false | ||
| - name: Copy _base folder and .env | ||
| shell: bash | ||
| run: | | ||
| if [[ -d "./deploy/build/_base" ]]; then | ||
| echo "copy _base directory into flavor" | ||
| cp -r -n "./deploy/build/_base/." "${{ matrix.component.flavor_directory }}" | ||
| tree "${{ matrix.component.flavor_directory }}" | ||
| fi | ||
| if [[ -f "${{ matrix.component.flavor_directory }}/${{ matrix.component.directory }}/.env" ]]; then | ||
| echo "copy .env into repo root" | ||
| cp "${{ matrix.component.flavor_directory }}/${{ matrix.component.directory }}/.env" "./" | ||
| fi | ||
| # Required for build secrets to work: https://docs.docker.com/build/ci/github-actions/secrets/#secret-mounts | ||
| - name: Set up QEMU | ||
| uses: docker/setup-qemu-action@v3 | ||
| - name: Set up Docker Buildx | ||
| uses: docker/setup-buildx-action@v3 | ||
| - name: Configure AWS Credentials | ||
| uses: aws-actions/[email protected] | ||
| with: | ||
| role-to-assume: ${{ vars.DV_AWS_ECR_ROLE }} | ||
| aws-region: ${{ vars.DV_AWS_REGION }} | ||
| - name: Login to Amazon ECR | ||
| id: login-ecr | ||
| uses: aws-actions/[email protected] | ||
| - uses: ./tmp/github-workflows/.github/actions/get-branch | ||
| id: get-branch | ||
| with: | ||
| branch: ${{ inputs.branch }} | ||
| - name: Build image | ||
| uses: docker/build-push-action@v6 | ||
| with: | ||
| context: . | ||
| file: ${{ matrix.component.flavor_directory }}/${{ matrix.component.directory }}/Dockerfile | ||
| push: false | ||
| load: true | ||
| # Disable provenance as it creates weird multi-arch images: https://github.com/docker/build-push-action/issues/755 | ||
| provenance: false | ||
| # Disable the cache to avoid outdated (base) images | ||
| no-cache: true | ||
| build-args: | | ||
| GIT_BRANCH=${{ steps.get-branch.outputs.branch }} | ||
| GIT_COMMIT_HASH=${{ steps.get-branch.outputs.commit_hash }} | ||
| DOCKERFILE_DIRECTORY=${{ matrix.component.flavor_directory }}/${{ matrix.component.directory }} | ||
| DATAVISYN_PYTHON_BASE_IMAGE=${{ env.DATAVISYN_PYTHON_BASE_IMAGE }} | ||
| DATAVISYN_NGINX_BASE_IMAGE=${{ env.DATAVISYN_NGINX_BASE_IMAGE }} | ||
| UV_HTTP_TIMEOUT=300 | ||
| ${{ matrix.component.formatted_build_args }} | ||
| secrets: | ||
| # Mount the token as secret mount: https://docs.docker.com/build/ci/github-actions/secrets/#secret-mounts | ||
| "github_token=${{ secrets.CHECKOUT_TOKEN || github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }}" | ||
| # TODO: As soon as we only have a single tag, we can push the same image to multiple repositories: https://docs.docker.com/build/ci/github-actions/push-multi-registries/ | ||
| # This will be useful for the images which don't change between flavors, e.g. the backend images | ||
| tags: | | ||
| ${{ matrix.component.image_ref }} | ||
| labels: | | ||
| name=${{ matrix.component.ecr_repository }} | ||
| version=${{ matrix.component.image_tag_branch_name }} | ||
| org.opencontainers.image.description=Image for ${{ matrix.component.ecr_repository }} | ||
| org.opencontainers.image.source=${{ github.event.repository.html_url }} | ||
| org.opencontainers.image.url=${{ github.event.repository.html_url }} | ||
| org.opencontainers.image.title=${{ matrix.component.ecr_repository }} | ||
| org.opencontainers.image.version=${{ matrix.component.image_tag_branch_name }} | ||
| org.opencontainers.image.created=${{ matrix.component.build_time }} | ||
| org.opencontainers.image.revision=${{ github.sha }} | ||
| env: | ||
| # Disable the build summary for now as it leads to "Failed to export build record: .../export/rec.dockerbuild not found" | ||
| # https://github.com/docker/build-push-action/issues/1156#issuecomment-2437227730 | ||
| DOCKER_BUILD_SUMMARY: false | ||
| - name: Determine trivy scan severity levels | ||
| id: set_severity | ||
| run: | | ||
| if [[ "${{ github.event.inputs.scan_high_severity }}" == "false" ]] || \ | ||
| [[ "${{ vars.SCAN_HIGH_SEVERITY }}" == "false" ]] || \ | ||
| [[ "${{ matrix.component.scan_high_severity }}" == "false" ]]; then | ||
| echo "severity=CRITICAL" >> "$GITHUB_OUTPUT" | ||
| else | ||
| echo "severity=HIGH,CRITICAL" >> "$GITHUB_OUTPUT" | ||
| fi | ||
| - name: Run Trivy vulnerability scanner | ||
| uses: aquasecurity/[email protected] | ||
| with: | ||
| image-ref: ${{ matrix.component.image_ref }} | ||
| # Disable scanning the current directory (defaults to .) | ||
| scan-ref: "/dev/null" | ||
| format: "table" | ||
| exit-code: "1" | ||
| ignore-unfixed: false | ||
| vuln-type: "os,library" | ||
| severity: ${{ steps.set_severity.outputs.severity }} | ||
| continue-on-error: false | ||
| - name: Push image | ||
| if: ${{ matrix.component.skip_push != true }} | ||
| # Instead of the docker/build-push-action@v6 which will rebuild the image, just push it directly | ||
| run: docker push ${{ matrix.component.image_ref }} | ||
| - name: Log out from Amazon ECR | ||
| shell: bash | ||
| run: docker logout ${{ steps.login-ecr.outputs.registry }} | ||
| test-images: | ||
| name: Test images of flavor ${{ matrix.flavor.id || 'default' }} | ||
| needs: [get-flavors, build-flavors] | ||
| if: ${{ matrix.flavor.test_images.enabled == true }} | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| flavor: ${{ fromJson(needs.get-flavors.outputs.result).flavors }} | ||
| runs-on: ${{ inputs.runs_on || 'ubuntu-22.04' }} | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v5 | ||
| with: | ||
| ref: ${{ inputs.branch || github.sha }} | ||
| token: ${{ secrets.CHECKOUT_TOKEN || github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} | ||
| - name: Configure AWS Credentials | ||
| uses: aws-actions/[email protected] | ||
| with: | ||
| role-to-assume: ${{ vars.DV_AWS_ECR_ROLE }} | ||
| aws-region: ${{ vars.DV_AWS_REGION }} | ||
| - name: Login to Amazon ECR | ||
| id: login-ecr | ||
| uses: aws-actions/[email protected] | ||
| - name: Run test-images.sh hook | ||
| shell: bash | ||
| id: test-images | ||
| run: | | ||
| test_images_hook=$(jq -r '.test_images.script_path' <<< "$FLAVOR") | ||
| test_images_report=$(jq -r '.test_images.report_path' <<< "$FLAVOR") | ||
| if [[ -f "$test_images_hook" ]]; then | ||
| # Iterate through all components and store their image ref in an environment variable | ||
| for component in $(jq -c '.components[]' <<< "$FLAVOR"); do | ||
| name=$(jq -r '.ecr_repository' <<< "$component") | ||
| image_ref=$(jq -r '.image_ref' <<< "$component") | ||
| # Replace all non-alphanumeric characters with underscores and convert to uppercase | ||
| name_upper=$(echo "${name//[^[:alnum:]]/_}" | tr '[:lower:]' '[:upper:]') | ||
| echo "Setting environment variable IMAGE_${name_upper}=${image_ref}" | ||
| export "IMAGE_${name_upper}=${image_ref}" | ||
| done; | ||
| # Create report folder to avoid any downstream Docker volume issues | ||
| # TODO: For some reason this doesn't work yet, i.e. if a docker-compose script mounts a volume here, nothing shows up... | ||
| mkdir -p "$test_images_report" | ||
| chmod 777 "$test_images_report" | ||
| echo "test_images_report=${test_images_report}" >> "$GITHUB_OUTPUT" | ||
| echo "Run $test_images_hook" | ||
| chmod +x "$test_images_hook" | ||
| bash "$test_images_hook" | ||
| else | ||
| echo "No $test_images_hook found, skipping tests." | ||
| fi | ||
| env: | ||
| FLAVOR: ${{ toJSON(matrix.flavor) }} | ||
| - name: Upload test-images-report | ||
| uses: actions/upload-artifact@v4 | ||
| if: ${{ steps.test-images.outputs.test_images_report }} | ||
| with: | ||
| name: "test-images-report-${{ matrix.flavor.id || 'default' }}" | ||
| path: ${{ steps.test-images.outputs.test_images_report }} | ||
| - name: Log out from Amazon ECR | ||
| shell: bash | ||
| run: docker logout ${{ steps.login-ecr.outputs.registry }} | ||
| retag-images: | ||
| name: Retag images of flavor ${{ matrix.flavor.id || 'default' }} | ||
| needs: [get-flavors, test-images] | ||
| # As test-images may be skipped, we need this if to ensure this job runs: https://github.com/actions/runner/issues/491#issuecomment-1507495166 | ||
| if: ${{ always() && !cancelled() && !failure() && matrix.flavor.skip_retag != true }} | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| flavor: ${{ fromJson(needs.get-flavors.outputs.result).flavors }} | ||
| # Do not run this on self-hosted, as it is faster and shouldn't be blocking anything | ||
| # runs-on: ${{ inputs.runs_on || 'ubuntu-22.04' }} | ||
| runs-on: "ubuntu-22.04" | ||
| steps: | ||
| - name: Checkout repository | ||
| uses: actions/checkout@v5 | ||
| with: | ||
| ref: ${{ inputs.branch || github.sha }} | ||
| token: ${{ secrets.CHECKOUT_TOKEN || github.event.repository.private == true && secrets.DATAVISYN_BOT_REPO_TOKEN || github.token }} | ||
| - name: Checkout github-workflows repository | ||
| uses: actions/checkout@v5 | ||
| with: | ||
| repository: datavisyn/github-workflows | ||
| ref: ${{ env.WORKFLOW_BRANCH }} | ||
| path: ./tmp/github-workflows | ||
| - name: Configure AWS Credentials | ||
| uses: aws-actions/[email protected] | ||
| with: | ||
| role-to-assume: ${{ vars.DV_AWS_ECR_ROLE }} | ||
| aws-region: ${{ vars.DV_AWS_REGION }} | ||
| - name: Login to Amazon ECR | ||
| id: login-ecr | ||
| uses: aws-actions/[email protected] | ||
| - name: Retag images | ||
| shell: bash | ||
| run: | | ||
| image_tag="${{ matrix.flavor.image_tag }}" | ||
| image_tag_branch_name="${{ matrix.flavor.image_tag_branch_name }}" | ||
| echo "image_tag=$image_tag" | ||
| echo "image_tag_branch_name=$image_tag_branch_name" | ||
| for repository_name in $(jq -r '.ecr_respositories[]' <<< "$FLAVOR"); do | ||
| IMAGE_META=$(aws ecr describe-images --repository-name "$repository_name" --image-ids imageTag="$image_tag" --output json | jq --arg var "${image_tag_branch_name}" '.imageDetails[0].imageTags | index( $var )') | ||
| if [[ -z "${IMAGE_META}" || ${IMAGE_META} == "null" ]]; then | ||
| MANIFEST=$(aws ecr batch-get-image --repository-name "$repository_name" --image-ids imageTag="$image_tag" --output json | jq --raw-output --join-output '.images[0].imageManifest') | ||
| aws ecr put-image --repository-name "$repository_name" --image-tag "$image_tag_branch_name" --image-manifest "$MANIFEST" | ||
| else | ||
| echo "image already tagged!" | ||
| fi | ||
| done; | ||
| env: | ||
| FLAVOR: ${{ toJSON(matrix.flavor) }} | ||
| - name: Log out from Amazon ECR | ||
| shell: bash | ||
| run: docker logout ${{ steps.login-ecr.outputs.registry }} | ||
| push-to-repositories: | ||
| name: Push images to push targets | ||
| # if? When should we do this? Always? Only for certain branches? If so, how should we define that, in the config.json? | ||
| if: ${{ fromJson(needs.get-flavors.outputs.result).push_to != '' }} | ||
| needs: [retag-images, get-flavors] | ||
| uses: datavisyn/github-workflows/.github/workflows/build-docker-artifacts-trigger-push.yml@main | ||
| secrets: inherit | ||
| with: | ||
| push_to: ${{ fromJson(needs.get-flavors.outputs.result).push_to }} | ||
| branch: ${{ inputs.branch || github.sha }} | ||
| # Do not run this on self-hosted, as it is faster and shouldn't be blocking anything | ||
| # runs_on: ${{ inputs.runs_on || 'ubuntu-22.04' }} | ||
