diff --git a/.github/workflows/build_spark-connect-client.yaml b/.github/workflows/build_spark-connect-client.yaml new file mode 100644 index 000000000..a8ec80123 --- /dev/null +++ b/.github/workflows/build_spark-connect-client.yaml @@ -0,0 +1,171 @@ +--- +name: Build Spark Connect Client +run-name: | + Build Spark Connect Client (attempt #${{ github.run_attempt }}) + +env: + PRODUCT_NAME: spark-connect-client + SDP_VERSION: ${{ github.ref_type == 'tag' && github.ref_name || '0.0.0-dev' }} + +on: + workflow_dispatch: + schedule: + - cron: '0 0 2/2 * *' # https://crontab.guru/#0_0_2/2_*_* + push: + branches: [main] + tags: ['*'] + paths: + # To check dependencies, run this ( you will need to consider transitive dependencies) + # bake --product PRODUCT -d | grep -v 'docker buildx bake' | jq '.target | keys[]' + - spark-k8s/** + - spark-connect-client/** + - stackable-base/** + - java-base/** + - .github/actions/** + - .github/workflows/build_spark-connect-client.yaml + +jobs: + generate_matrix: + name: Generate Version List + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + persist-credentials: false + - id: shard + uses: stackabletech/actions/shard@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0 + with: + product-name: ${{ env.PRODUCT_NAME }} + outputs: + versions: ${{ steps.shard.outputs.versions }} + + build: + name: Build/Publish ${{ matrix.versions }}-${{ matrix.runner.arch }} Image + needs: [generate_matrix] + permissions: + id-token: write + runs-on: ${{ matrix.runner.name }} + strategy: + fail-fast: false + matrix: + runner: + - {name: "ubuntu-latest", arch: "amd64"} + - {name: "ubicloud-standard-8-arm", arch: "arm64"} + versions: ${{ fromJson(needs.generate_matrix.outputs.versions) }} + steps: + - name: Checkout Repository + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + persist-credentials: false + + - name: Free Disk Space + uses: stackabletech/actions/free-disk-space@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0 + + - name: Build Product Image + id: build + uses: stackabletech/actions/build-product-image@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0 + with: + product-name: ${{ env.PRODUCT_NAME }} + product-version: ${{ matrix.versions }} + build-cache-password: ${{ secrets.BUILD_CACHE_NEXUS_PASSWORD }} + sdp-version: ${{ env.SDP_VERSION }} + + - name: Publish Container Image on docker.stackable.tech + uses: stackabletech/actions/publish-image@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0 + with: + image-registry-uri: docker.stackable.tech + image-registry-username: github + image-registry-password: ${{ secrets.NEXUS_PASSWORD }} + image-repository: stackable/${{ env.PRODUCT_NAME }} + image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }} + source-image-uri: localhost/${{ env.PRODUCT_NAME }}:${{ steps.build.outputs.image-manifest-tag }} + + - name: Publish Container Image on oci.stackable.tech + uses: stackabletech/actions/publish-image@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0 + with: + image-registry-uri: oci.stackable.tech + image-registry-username: robot$sdp+github-action-build + image-registry-password: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }} + image-repository: sdp/${{ env.PRODUCT_NAME }} + image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }} + source-image-uri: localhost/${{ env.PRODUCT_NAME }}:${{ steps.build.outputs.image-manifest-tag }} + + publish_manifests: + name: Build/Publish ${{ matrix.versions }} Manifests + needs: [generate_matrix, build] + permissions: + id-token: write + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + versions: ${{ fromJson(needs.generate_matrix.outputs.versions) }} + steps: + - name: Checkout Repository + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + persist-credentials: false + + - name: Publish and Sign Image Index Manifest to docker.stackable.tech + uses: stackabletech/actions/publish-index-manifest@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0 + with: + image-registry-uri: docker.stackable.tech + image-registry-username: github + image-registry-password: ${{ secrets.NEXUS_PASSWORD }} + image-repository: stackable/${{ env.PRODUCT_NAME }} + image-index-manifest-tag: ${{ matrix.versions }}-stackable${{ env.SDP_VERSION }} + + - name: Publish and Sign Image Index Manifest to oci.stackable.tech + uses: stackabletech/actions/publish-index-manifest@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0 + with: + image-registry-uri: oci.stackable.tech + image-registry-username: robot$sdp+github-action-build + image-registry-password: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }} + image-repository: sdp/${{ env.PRODUCT_NAME }} + image-index-manifest-tag: ${{ matrix.versions }}-stackable${{ env.SDP_VERSION }} + + notify: + name: Failure Notification + needs: [generate_matrix, build, publish_manifests] + runs-on: ubuntu-latest + if: failure() + steps: + - uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0 + with: + channel-id: "C07UG6JH44F" # notifications-container-images + payload: | + { + "text": "*${{ github.workflow }}* failed (attempt ${{ github.run_attempt }})", + "attachments": [ + { + "pretext": "See the details below for a summary of which job(s) failed.", + "color": "#aa0000", + "fields": [ + { + "title": "Generate Version List", + "short": true, + "value": "${{ needs.generate_matrix.result }}" + }, + { + "title": "Build/Publish Image", + "short": true, + "value": "${{ needs.build.result }}" + }, + { + "title": "Build/Publish Manifests", + "short": true, + "value": "${{ needs.publish_manifests.result }}" + } + ], + "actions": [ + { + "type": "button", + "text": "Go to workflow run", + "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}" + } + ] + } + ] + } + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_CONTAINER_IMAGE_TOKEN }} diff --git a/.scripts/update_readme_badges.sh b/.scripts/update_readme_badges.sh index 8df0f40b0..140b214ba 100755 --- a/.scripts/update_readme_badges.sh +++ b/.scripts/update_readme_badges.sh @@ -54,6 +54,10 @@ for BUILD_WORKFLOW_FILE in .github/workflows/build_*.yaml; do echo >> "$BADGES_TMP" fi done +# This needs to add the remaning empty columns of the last row in the table +# This is a hack to fix the status quo and make markdownlint happy. +echo -n "| | | |" >> "$BADGES_TMP" +echo >> "$BADGES_TMP" echo -n "" >> "$BADGES_TMP" # Print the image and link shortcuts. Eg: diff --git a/README.md b/README.md index 4e3b6f3d7..c1a38b8e4 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,10 @@ This repository contains Dockerfiles and scripts to build base images for use wi | [![Build Airflow]][build_airflow.yaml] | [![Build Druid]][build_druid.yaml] | [![Build Hadoop]][build_hadoop.yaml] | [![Build HBase]][build_hbase.yaml] | | [![Build Hello-World]][build_hello-world.yaml] | [![Build Hive]][build_hive.yaml] | [![Build Java Base]][build_java-base.yaml] | [![Build Java Development]][build_java-devel.yaml] | | [![Build Kafka Testing Tools]][build_kafka-testing-tools.yaml] | [![Build Kafka]][build_kafka.yaml] | [![Build kcat]][build_kcat.yaml] | [![Build Krb5]][build_krb5.yaml] | -| [![Build NiFi]][build_nifi.yaml] | [![Build Omid]][build_omid.yaml] | [![Build OPA]][build_opa.yaml] | [![Build Spark K8s]][build_spark-k8s.yaml] | -| [![Build Stackable Base]][build_stackable-base.yaml] | [![Build Superset]][build_superset.yaml] | [![Build Testing Tools]][build_testing-tools.yaml] | [![Build Tools]][build_tools.yaml] | -| [![Build Trino CLI]][build_trino-cli.yaml] | [![Build Trino]][build_trino.yaml] | [![Build Vector]][build_vector.yaml] | [![Build ZooKeeper]][build_zookeeper.yaml] | +| [![Build NiFi]][build_nifi.yaml] | [![Build Omid]][build_omid.yaml] | [![Build OPA]][build_opa.yaml] | [![Build Spark Connect Client]][build_spark-connect-client.yaml] | +| [![Build Spark K8s]][build_spark-k8s.yaml] | [![Build Stackable Base]][build_stackable-base.yaml] | [![Build Superset]][build_superset.yaml] | [![Build Testing Tools]][build_testing-tools.yaml] | +| [![Build Tools]][build_tools.yaml] | [![Build Trino CLI]][build_trino-cli.yaml] | [![Build Trino]][build_trino.yaml] | [![Build Vector]][build_vector.yaml] | +| [![Build ZooKeeper]][build_zookeeper.yaml] | | | | ## Prerequisites @@ -243,6 +244,8 @@ ENTRYPOINT ["/stackable-zookeeper-operator"] [build_omid.yaml]: https://github.com/stackabletech/docker-images/actions/workflows/build_omid.yaml [Build OPA]: https://github.com/stackabletech/docker-images/actions/workflows/build_opa.yaml/badge.svg [build_opa.yaml]: https://github.com/stackabletech/docker-images/actions/workflows/build_opa.yaml +[Build Spark Connect Client]: https://github.com/stackabletech/docker-images/actions/workflows/build_spark-connect-client.yaml/badge.svg +[build_spark-connect-client.yaml]: https://github.com/stackabletech/docker-images/actions/workflows/build_spark-connect-client.yaml [Build Spark K8s]: https://github.com/stackabletech/docker-images/actions/workflows/build_spark-k8s.yaml/badge.svg [build_spark-k8s.yaml]: https://github.com/stackabletech/docker-images/actions/workflows/build_spark-k8s.yaml [Build Stackable Base]: https://github.com/stackabletech/docker-images/actions/workflows/build_stackable-base.yaml/badge.svg