Skip to content

Commit b9124a1

Browse files
razvanrazvan
committed
feat: add spark connect client to image build flows
1 parent a317716 commit b9124a1

File tree

3 files changed

+185
-3
lines changed

3 files changed

+185
-3
lines changed

‎.github/workflows/build_spark-connect-client.yaml‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
---
2+
name: Build Spark Connect Client
3+
run-name: |
4+
Build Spark Connect Client (attempt #${{ github.run_attempt }})
5+
6+
env:
7+
PRODUCT_NAME: spark-connect-client
8+
SDP_VERSION: ${{ github.ref_type == 'tag' && github.ref_name || '0.0.0-dev' }}
9+
10+
on:
11+
workflow_dispatch:
12+
schedule:
13+
- cron: '0 0 2/2 * *' # https://crontab.guru/#0_0_2/2_*_*
14+
push:
15+
branches: [main]
16+
tags: ['*']
17+
paths:
18+
# To check dependencies, run this ( you will need to consider transitive dependencies)
19+
# bake --product PRODUCT -d | grep -v 'docker buildx bake' | jq '.target | keys[]'
20+
- hadoop/**
21+
- hbase/**
22+
- spark-k8s/**
23+
- spark-connect-client/**
24+
- vector/**
25+
- stackable-base/**
26+
- java-base/**
27+
- java-devel/**
28+
- .github/actions/**
29+
- .github/workflows/build_spark-k8s.yaml
30+
31+
jobs:
32+
generate_matrix:
33+
name: Generate Version List
34+
runs-on: ubuntu-latest
35+
steps:
36+
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
37+
with:
38+
persist-credentials: false
39+
- id: shard
40+
uses: stackabletech/actions/shard@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0
41+
with:
42+
product-name: ${{ env.PRODUCT_NAME }}
43+
outputs:
44+
versions: ${{ steps.shard.outputs.versions }}
45+
46+
build:
47+
name: Build/Publish ${{ matrix.versions }}-${{ matrix.runner.arch }} Image
48+
needs: [generate_matrix]
49+
permissions:
50+
id-token: write
51+
runs-on: ${{ matrix.runner.name }}
52+
strategy:
53+
fail-fast: false
54+
matrix:
55+
runner:
56+
- {name: "ubuntu-latest", arch: "amd64"}
57+
- {name: "ubicloud-standard-8-arm", arch: "arm64"}
58+
versions: ${{ fromJson(needs.generate_matrix.outputs.versions) }}
59+
steps:
60+
- name: Checkout Repository
61+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
62+
with:
63+
persist-credentials: false
64+
65+
- name: Free Disk Space
66+
uses: stackabletech/actions/free-disk-space@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0
67+
68+
- name: Build Product Image
69+
id: build
70+
uses: stackabletech/actions/build-product-image@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0
71+
with:
72+
product-name: ${{ env.PRODUCT_NAME }}
73+
product-version: ${{ matrix.versions }}
74+
build-cache-password: ${{ secrets.BUILD_CACHE_NEXUS_PASSWORD }}
75+
sdp-version: ${{ env.SDP_VERSION }}
76+
77+
- name: Publish Container Image on docker.stackable.tech
78+
uses: stackabletech/actions/publish-image@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0
79+
with:
80+
image-registry-uri: docker.stackable.tech
81+
image-registry-username: github
82+
image-registry-password: ${{ secrets.NEXUS_PASSWORD }}
83+
image-repository: stackable/${{ env.PRODUCT_NAME }}
84+
image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
85+
source-image-uri: localhost/${{ env.PRODUCT_NAME }}:${{ steps.build.outputs.image-manifest-tag }}
86+
87+
- name: Publish Container Image on oci.stackable.tech
88+
uses: stackabletech/actions/publish-image@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0
89+
with:
90+
image-registry-uri: oci.stackable.tech
91+
image-registry-username: robot$sdp+github-action-build
92+
image-registry-password: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }}
93+
image-repository: sdp/${{ env.PRODUCT_NAME }}
94+
image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
95+
source-image-uri: localhost/${{ env.PRODUCT_NAME }}:${{ steps.build.outputs.image-manifest-tag }}
96+
97+
publish_manifests:
98+
name: Build/Publish ${{ matrix.versions }} Manifests
99+
needs: [generate_matrix, build]
100+
permissions:
101+
id-token: write
102+
runs-on: ubuntu-latest
103+
strategy:
104+
fail-fast: false
105+
matrix:
106+
versions: ${{ fromJson(needs.generate_matrix.outputs.versions) }}
107+
steps:
108+
- name: Checkout Repository
109+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
110+
with:
111+
persist-credentials: false
112+
113+
- name: Publish and Sign Image Index Manifest to docker.stackable.tech
114+
uses: stackabletech/actions/publish-index-manifest@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0
115+
with:
116+
image-registry-uri: docker.stackable.tech
117+
image-registry-username: github
118+
image-registry-password: ${{ secrets.NEXUS_PASSWORD }}
119+
image-repository: stackable/${{ env.PRODUCT_NAME }}
120+
image-index-manifest-tag: ${{ matrix.versions }}-stackable${{ env.SDP_VERSION }}
121+
122+
- name: Publish and Sign Image Index Manifest to oci.stackable.tech
123+
uses: stackabletech/actions/publish-index-manifest@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0
124+
with:
125+
image-registry-uri: oci.stackable.tech
126+
image-registry-username: robot$sdp+github-action-build
127+
image-registry-password: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }}
128+
image-repository: sdp/${{ env.PRODUCT_NAME }}
129+
image-index-manifest-tag: ${{ matrix.versions }}-stackable${{ env.SDP_VERSION }}
130+
131+
notify:
132+
name: Failure Notification
133+
needs: [generate_matrix, build, publish_manifests]
134+
runs-on: ubuntu-latest
135+
if: failure()
136+
steps:
137+
- uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
138+
with:
139+
channel-id: "C07UG6JH44F" # notifications-container-images
140+
payload: |
141+
{
142+
"text": "*${{ github.workflow }}* failed (attempt ${{ github.run_attempt }})",
143+
"attachments": [
144+
{
145+
"pretext": "See the details below for a summary of which job(s) failed.",
146+
"color": "#aa0000",
147+
"fields": [
148+
{
149+
"title": "Generate Version List",
150+
"short": true,
151+
"value": "${{ needs.generate_matrix.result }}"
152+
},
153+
{
154+
"title": "Build/Publish Image",
155+
"short": true,
156+
"value": "${{ needs.build.result }}"
157+
},
158+
{
159+
"title": "Build/Publish Manifests",
160+
"short": true,
161+
"value": "${{ needs.publish_manifests.result }}"
162+
}
163+
],
164+
"actions": [
165+
{
166+
"type": "button",
167+
"text": "Go to workflow run",
168+
"url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}"
169+
}
170+
]
171+
}
172+
]
173+
}
174+
env:
175+
SLACK_BOT_TOKEN: ${{ secrets.SLACK_CONTAINER_IMAGE_TOKEN }}

‎.scripts/update_readme_badges.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,10 @@ for BUILD_WORKFLOW_FILE in .github/workflows/build_*.yaml; do
5454
echo >> "$BADGES_TMP"
5555
fi
5656
done
57+
# This needs to add the remaning empty columns of the last row in the table
58+
# This is a hack to fix the status quo and make markdownlint happy.
59+
echo -n "| | | |" >> "$BADGES_TMP"
60+
echo >> "$BADGES_TMP"
5761
echo -n "<!-- end:badges -->" >> "$BADGES_TMP"
5862

5963
# Print the image and link shortcuts. Eg:

‎README.md‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,10 @@ This repository contains Dockerfiles and scripts to build base images for use wi
88
| [![Build Airflow]][build_airflow.yaml] | [![Build Druid]][build_druid.yaml] | [![Build Hadoop]][build_hadoop.yaml] | [![Build HBase]][build_hbase.yaml] |
99
| [![Build Hello-World]][build_hello-world.yaml] | [![Build Hive]][build_hive.yaml] | [![Build Java Base]][build_java-base.yaml] | [![Build Java Development]][build_java-devel.yaml] |
1010
| [![Build Kafka Testing Tools]][build_kafka-testing-tools.yaml] | [![Build Kafka]][build_kafka.yaml] | [![Build kcat]][build_kcat.yaml] | [![Build Krb5]][build_krb5.yaml] |
11-
| [![Build NiFi]][build_nifi.yaml] | [![Build Omid]][build_omid.yaml] | [![Build OPA]][build_opa.yaml] | [![Build Spark K8s]][build_spark-k8s.yaml] |
12-
| [![Build Stackable Base]][build_stackable-base.yaml] | [![Build Superset]][build_superset.yaml] | [![Build Testing Tools]][build_testing-tools.yaml] | [![Build Tools]][build_tools.yaml] |
13-
| [![Build Trino CLI]][build_trino-cli.yaml] | [![Build Trino]][build_trino.yaml] | [![Build Vector]][build_vector.yaml] | [![Build ZooKeeper]][build_zookeeper.yaml] |
11+
| [![Build NiFi]][build_nifi.yaml] | [![Build Omid]][build_omid.yaml] | [![Build OPA]][build_opa.yaml] | [![Build Spark Connect Client]][build_spark-connect-client.yaml] |
12+
| [![Build Spark K8s]][build_spark-k8s.yaml] | [![Build Stackable Base]][build_stackable-base.yaml] | [![Build Superset]][build_superset.yaml] | [![Build Testing Tools]][build_testing-tools.yaml] |
13+
| [![Build Tools]][build_tools.yaml] | [![Build Trino CLI]][build_trino-cli.yaml] | [![Build Trino]][build_trino.yaml] | [![Build Vector]][build_vector.yaml] |
14+
| [![Build ZooKeeper]][build_zookeeper.yaml] | | | |
1415
<!-- end:badges -->
1516

1617
## Prerequisites
@@ -243,6 +244,8 @@ ENTRYPOINT ["/stackable-zookeeper-operator"]
243244
[build_omid.yaml]: https://github.com/stackabletech/docker-images/actions/workflows/build_omid.yaml
244245
[Build OPA]: https://github.com/stackabletech/docker-images/actions/workflows/build_opa.yaml/badge.svg
245246
[build_opa.yaml]: https://github.com/stackabletech/docker-images/actions/workflows/build_opa.yaml
247+
[Build Spark Connect Client]: https://github.com/stackabletech/docker-images/actions/workflows/build_spark-connect-client.yaml/badge.svg
248+
[build_spark-connect-client.yaml]: https://github.com/stackabletech/docker-images/actions/workflows/build_spark-connect-client.yaml
246249
[Build Spark K8s]: https://github.com/stackabletech/docker-images/actions/workflows/build_spark-k8s.yaml/badge.svg
247250
[build_spark-k8s.yaml]: https://github.com/stackabletech/docker-images/actions/workflows/build_spark-k8s.yaml
248251
[Build Stackable Base]: https://github.com/stackabletech/docker-images/actions/workflows/build_stackable-base.yaml/badge.svg

0 commit comments

Comments
 (0)