Skip to content

Commit e3fc4bd

Browse files
maltesandermaltesander
committed
Merge remote-tracking branch 'origin/main' into fix/hive-reduce-image-size
2 parents 9f49b2c + fe6b07a commit e3fc4bd

File tree

12 files changed

+637
-326
lines changed

12 files changed

+637
-326
lines changed

‎.github/workflows/build_spark-connect-client.yaml‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
---
2+
name: Build Spark Connect Client
3+
run-name: |
4+
Build Spark Connect Client (attempt #${{ github.run_attempt }})
5+
6+
env:
7+
PRODUCT_NAME: spark-connect-client
8+
SDP_VERSION: ${{ github.ref_type == 'tag' && github.ref_name || '0.0.0-dev' }}
9+
10+
on:
11+
workflow_dispatch:
12+
schedule:
13+
- cron: '0 0 2/2 * *' # https://crontab.guru/#0_0_2/2_*_*
14+
push:
15+
branches: [main]
16+
tags: ['*']
17+
paths:
18+
# To check dependencies, run this ( you will need to consider transitive dependencies)
19+
# bake --product PRODUCT -d | grep -v 'docker buildx bake' | jq '.target | keys[]'
20+
- spark-k8s/**
21+
- spark-connect-client/**
22+
- stackable-base/**
23+
- java-base/**
24+
- .github/actions/**
25+
- .github/workflows/build_spark-connect-client.yaml
26+
27+
jobs:
28+
generate_matrix:
29+
name: Generate Version List
30+
runs-on: ubuntu-latest
31+
steps:
32+
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
33+
with:
34+
persist-credentials: false
35+
- id: shard
36+
uses: stackabletech/actions/shard@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0
37+
with:
38+
product-name: ${{ env.PRODUCT_NAME }}
39+
outputs:
40+
versions: ${{ steps.shard.outputs.versions }}
41+
42+
build:
43+
name: Build/Publish ${{ matrix.versions }}-${{ matrix.runner.arch }} Image
44+
needs: [generate_matrix]
45+
permissions:
46+
id-token: write
47+
runs-on: ${{ matrix.runner.name }}
48+
strategy:
49+
fail-fast: false
50+
matrix:
51+
runner:
52+
- {name: "ubuntu-latest", arch: "amd64"}
53+
- {name: "ubicloud-standard-8-arm", arch: "arm64"}
54+
versions: ${{ fromJson(needs.generate_matrix.outputs.versions) }}
55+
steps:
56+
- name: Checkout Repository
57+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
58+
with:
59+
persist-credentials: false
60+
61+
- name: Free Disk Space
62+
uses: stackabletech/actions/free-disk-space@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0
63+
64+
- name: Build Product Image
65+
id: build
66+
uses: stackabletech/actions/build-product-image@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0
67+
with:
68+
product-name: ${{ env.PRODUCT_NAME }}
69+
product-version: ${{ matrix.versions }}
70+
build-cache-password: ${{ secrets.BUILD_CACHE_NEXUS_PASSWORD }}
71+
sdp-version: ${{ env.SDP_VERSION }}
72+
73+
- name: Publish Container Image on docker.stackable.tech
74+
uses: stackabletech/actions/publish-image@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0
75+
with:
76+
image-registry-uri: docker.stackable.tech
77+
image-registry-username: github
78+
image-registry-password: ${{ secrets.NEXUS_PASSWORD }}
79+
image-repository: stackable/${{ env.PRODUCT_NAME }}
80+
image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
81+
source-image-uri: localhost/${{ env.PRODUCT_NAME }}:${{ steps.build.outputs.image-manifest-tag }}
82+
83+
- name: Publish Container Image on oci.stackable.tech
84+
uses: stackabletech/actions/publish-image@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0
85+
with:
86+
image-registry-uri: oci.stackable.tech
87+
image-registry-username: robot$sdp+github-action-build
88+
image-registry-password: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }}
89+
image-repository: sdp/${{ env.PRODUCT_NAME }}
90+
image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
91+
source-image-uri: localhost/${{ env.PRODUCT_NAME }}:${{ steps.build.outputs.image-manifest-tag }}
92+
93+
publish_manifests:
94+
name: Build/Publish ${{ matrix.versions }} Manifests
95+
needs: [generate_matrix, build]
96+
permissions:
97+
id-token: write
98+
runs-on: ubuntu-latest
99+
strategy:
100+
fail-fast: false
101+
matrix:
102+
versions: ${{ fromJson(needs.generate_matrix.outputs.versions) }}
103+
steps:
104+
- name: Checkout Repository
105+
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
106+
with:
107+
persist-credentials: false
108+
109+
- name: Publish and Sign Image Index Manifest to docker.stackable.tech
110+
uses: stackabletech/actions/publish-index-manifest@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0
111+
with:
112+
image-registry-uri: docker.stackable.tech
113+
image-registry-username: github
114+
image-registry-password: ${{ secrets.NEXUS_PASSWORD }}
115+
image-repository: stackable/${{ env.PRODUCT_NAME }}
116+
image-index-manifest-tag: ${{ matrix.versions }}-stackable${{ env.SDP_VERSION }}
117+
118+
- name: Publish and Sign Image Index Manifest to oci.stackable.tech
119+
uses: stackabletech/actions/publish-index-manifest@320eae677555385b3d40e1c3a81d9263b72742e4 # 0.6.0
120+
with:
121+
image-registry-uri: oci.stackable.tech
122+
image-registry-username: robot$sdp+github-action-build
123+
image-registry-password: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }}
124+
image-repository: sdp/${{ env.PRODUCT_NAME }}
125+
image-index-manifest-tag: ${{ matrix.versions }}-stackable${{ env.SDP_VERSION }}
126+
127+
notify:
128+
name: Failure Notification
129+
needs: [generate_matrix, build, publish_manifests]
130+
runs-on: ubuntu-latest
131+
if: failure()
132+
steps:
133+
- uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
134+
with:
135+
channel-id: "C07UG6JH44F" # notifications-container-images
136+
payload: |
137+
{
138+
"text": "*${{ github.workflow }}* failed (attempt ${{ github.run_attempt }})",
139+
"attachments": [
140+
{
141+
"pretext": "See the details below for a summary of which job(s) failed.",
142+
"color": "#aa0000",
143+
"fields": [
144+
{
145+
"title": "Generate Version List",
146+
"short": true,
147+
"value": "${{ needs.generate_matrix.result }}"
148+
},
149+
{
150+
"title": "Build/Publish Image",
151+
"short": true,
152+
"value": "${{ needs.build.result }}"
153+
},
154+
{
155+
"title": "Build/Publish Manifests",
156+
"short": true,
157+
"value": "${{ needs.publish_manifests.result }}"
158+
}
159+
],
160+
"actions": [
161+
{
162+
"type": "button",
163+
"text": "Go to workflow run",
164+
"url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}"
165+
}
166+
]
167+
}
168+
]
169+
}
170+
env:
171+
SLACK_BOT_TOKEN: ${{ secrets.SLACK_CONTAINER_IMAGE_TOKEN }}

‎.scripts/update_readme_badges.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,10 @@ for BUILD_WORKFLOW_FILE in .github/workflows/build_*.yaml; do
5454
echo >> "$BADGES_TMP"
5555
fi
5656
done
57+
# This needs to add the remaning empty columns of the last row in the table
58+
# This is a hack to fix the status quo and make markdownlint happy.
59+
echo -n "| | | |" >> "$BADGES_TMP"
60+
echo >> "$BADGES_TMP"
5761
echo -n "<!-- end:badges -->" >> "$BADGES_TMP"
5862

5963
# Print the image and link shortcuts. Eg:

‎CHANGELOG.md‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,25 @@ All notable changes to this project will be documented in this file.
66

77
### Added
88

9+
- airflow: check for correct permissions and ownerships in /stackable folder via
10+
`check-permissions-ownership.sh` provided in stackable-base image ([#1054]).
11+
- druid: check for correct permissions and ownerships in /stackable folder via
12+
`check-permissions-ownership.sh` provided in stackable-base image ([#1039]).
13+
- hadoop: check for correct permissions and ownerships in /stackable folder via
14+
`check-permissions-ownership.sh` provided in stackable-base image ([#1029]).
15+
- hbase: check for correct permissions and ownerships in /stackable folder via
16+
`check-permissions-ownership.sh` provided in stackable-base image ([#1028]).
917
- hive: check for correct permissions and ownerships in /stackable folder via
1018
`check-permissions-ownership.sh` provided in stackable-base image ([#1040]).
1119
- spark-connect-client: A new image for Spark connect tests and demos ([#1034])
1220
- nifi: check for correct permissions and ownerships in /stackable folder via
1321
`check-permissions-ownership.sh` provided in stackable-base image ([#1027]).
22+
- opa: check for correct permissions and ownerships in /stackable folder via
23+
`check-permissions-ownership.sh` provided in stackable-base image ([#1038]).
24+
- superset: check for correct permissions and ownerships in /stackable folder via
25+
`check-permissions-ownership.sh` provided in stackable-base image ([#1053]).
26+
- trino: check for correct permissions and ownerships in /stackable folder via
27+
`check-permissions-ownership.sh` provided in stackable-base image ([#1025]).
1428

1529
### Changed
1630

@@ -19,17 +33,29 @@ All notable changes to this project will be documented in this file.
1933

2034
### Fixed
2135

36+
- druid: reduce docker image size by removing the recursive chown/chmods in the final image ([#1039]).
37+
- hadoop: reduce docker image size by removing the recursive chown/chmods in the final image ([#1029]).
38+
- hbase: reduce docker image size by removing the recursive chown/chmods in the final image ([#1028]).
2239
- hive: reduce docker image size by removing the recursive chown/chmods in the final image ([#1040]).
2340
- nifi: reduce docker image size by removing the recursive chown/chmods in the final image ([#1027]).
41+
- opa: reduce docker image size by removing the recursive chown/chmods in the final image ([#1038]).
2442
- spark-k8s: reduce docker image size by removing the recursive chown/chmods in the final image ([#1042]).
43+
- trino: reduce docker image size by removing the recursive chown/chmods in the final image ([#1025]).
2544
- Add `--locked` flag to `cargo install` commands for reproducible builds ([#1044]).
2645

46+
[#1025]: https://github.com/stackabletech/docker-images/pull/1025
2747
[#1027]: https://github.com/stackabletech/docker-images/pull/1027
48+
[#1028]: https://github.com/stackabletech/docker-images/pull/1028
49+
[#1029]: https://github.com/stackabletech/docker-images/pull/1029
2850
[#1034]: https://github.com/stackabletech/docker-images/pull/1034
51+
[#1038]: https://github.com/stackabletech/docker-images/pull/1038
52+
[#1039]: https://github.com/stackabletech/docker-images/pull/1039
2953
[#1040]: https://github.com/stackabletech/docker-images/pull/1040
3054
[#1042]: https://github.com/stackabletech/docker-images/pull/1042
3155
[#1044]: https://github.com/stackabletech/docker-images/pull/1044
3256
[#1050]: https://github.com/stackabletech/docker-images/pull/1050
57+
[#1053]: https://github.com/stackabletech/docker-images/pull/1053
58+
[#1054]: https://github.com/stackabletech/docker-images/pull/1054
3359

3460
## [25.3.0] - 2025-03-21
3561

‎README.md‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,10 @@ This repository contains Dockerfiles and scripts to build base images for use wi
88
| [![Build Airflow]][build_airflow.yaml] | [![Build Druid]][build_druid.yaml] | [![Build Hadoop]][build_hadoop.yaml] | [![Build HBase]][build_hbase.yaml] |
99
| [![Build Hello-World]][build_hello-world.yaml] | [![Build Hive]][build_hive.yaml] | [![Build Java Base]][build_java-base.yaml] | [![Build Java Development]][build_java-devel.yaml] |
1010
| [![Build Kafka Testing Tools]][build_kafka-testing-tools.yaml] | [![Build Kafka]][build_kafka.yaml] | [![Build kcat]][build_kcat.yaml] | [![Build Krb5]][build_krb5.yaml] |
11-
| [![Build NiFi]][build_nifi.yaml] | [![Build Omid]][build_omid.yaml] | [![Build OPA]][build_opa.yaml] | [![Build Spark K8s]][build_spark-k8s.yaml] |
12-
| [![Build Stackable Base]][build_stackable-base.yaml] | [![Build Superset]][build_superset.yaml] | [![Build Testing Tools]][build_testing-tools.yaml] | [![Build Tools]][build_tools.yaml] |
13-
| [![Build Trino CLI]][build_trino-cli.yaml] | [![Build Trino]][build_trino.yaml] | [![Build Vector]][build_vector.yaml] | [![Build ZooKeeper]][build_zookeeper.yaml] |
11+
| [![Build NiFi]][build_nifi.yaml] | [![Build Omid]][build_omid.yaml] | [![Build OPA]][build_opa.yaml] | [![Build Spark Connect Client]][build_spark-connect-client.yaml] |
12+
| [![Build Spark K8s]][build_spark-k8s.yaml] | [![Build Stackable Base]][build_stackable-base.yaml] | [![Build Superset]][build_superset.yaml] | [![Build Testing Tools]][build_testing-tools.yaml] |
13+
| [![Build Tools]][build_tools.yaml] | [![Build Trino CLI]][build_trino-cli.yaml] | [![Build Trino]][build_trino.yaml] | [![Build Vector]][build_vector.yaml] |
14+
| [![Build ZooKeeper]][build_zookeeper.yaml] | | | |
1415
<!-- end:badges -->
1516

1617
## Prerequisites
@@ -243,6 +244,8 @@ ENTRYPOINT ["/stackable-zookeeper-operator"]
243244
[build_omid.yaml]: https://github.com/stackabletech/docker-images/actions/workflows/build_omid.yaml
244245
[Build OPA]: https://github.com/stackabletech/docker-images/actions/workflows/build_opa.yaml/badge.svg
245246
[build_opa.yaml]: https://github.com/stackabletech/docker-images/actions/workflows/build_opa.yaml
247+
[Build Spark Connect Client]: https://github.com/stackabletech/docker-images/actions/workflows/build_spark-connect-client.yaml/badge.svg
248+
[build_spark-connect-client.yaml]: https://github.com/stackabletech/docker-images/actions/workflows/build_spark-connect-client.yaml
246249
[Build Spark K8s]: https://github.com/stackabletech/docker-images/actions/workflows/build_spark-k8s.yaml/badge.svg
247250
[build_spark-k8s.yaml]: https://github.com/stackabletech/docker-images/actions/workflows/build_spark-k8s.yaml
248251
[Build Stackable Base]: https://github.com/stackabletech/docker-images/actions/workflows/build_stackable-base.yaml/badge.svg

‎airflow/Dockerfile‎

Lines changed: 38 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ ARG PRODUCT
2828
ARG STATSD_EXPORTER
2929
ARG PYTHON
3030
ARG TARGETARCH
31+
ARG STACKABLE_USER_UID
3132

3233
COPY airflow/constraints-${PRODUCT}-python${PYTHON}.txt /tmp/constraints.txt
3334
COPY --from=opa-auth-manager-builder /tmp/opa-auth-manager/dist/opa_auth_manager-0.1.0-py3-none-any.whl /tmp/
@@ -85,9 +86,17 @@ else
8586
end)' /tmp/sbom.json > /stackable/app/airflow-${PRODUCT}.cdx.json
8687
EOF
8788

88-
WORKDIR /stackable
8989
COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter /stackable/statsd_exporter
9090
COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter-${STATSD_EXPORTER}.cdx.json /stackable/statsd_exporter-${STATSD_EXPORTER}.cdx.json
91+
COPY --from=gitsync-image --chown=${STACKABLE_USER_UID}:0 /git-sync /stackable/git-sync
92+
93+
RUN <<EOF
94+
mkdir -pv /stackable/airflow
95+
mkdir -pv /stackable/airflow/dags
96+
mkdir -pv /stackable/airflow/logs
97+
chmod --recursive g=u /stackable
98+
EOF
99+
91100

92101
FROM stackable/image/vector AS airflow-main-image
93102

@@ -99,22 +108,26 @@ ARG TARGETARCH
99108
ARG STACKABLE_USER_UID
100109

101110
LABEL name="Apache Airflow" \
102-
maintainer="[email protected]" \
103-
vendor="Stackable GmbH" \
104-
version="${PRODUCT}" \
105-
release="${RELEASE}" \
106-
summary="The Stackable image for Apache Airflow." \
107-
description="This image is deployed by the Stackable Operator for Apache Airflow."
108-
109-
COPY airflow/licenses /licenses
110-
COPY --chown=${STACKABLE_USER_UID}:0 airflow/stackable/utils/entrypoint.sh /entrypoint.sh
111-
COPY --chown=${STACKABLE_USER_UID}:0 airflow/stackable/utils/run-airflow.sh /run-airflow.sh
111+
maintainer="[email protected]" \
112+
vendor="Stackable GmbH" \
113+
version="${PRODUCT}" \
114+
release="${RELEASE}" \
115+
summary="The Stackable image for Apache Airflow." \
116+
description="This image is deployed by the Stackable Operator for Apache Airflow."
112117

113118
ENV HOME=/stackable
114119
ENV AIRFLOW_USER_HOME_DIR=/stackable
115120
ENV PATH=$PATH:/bin:$HOME/app/bin
116121
ENV AIRFLOW_HOME=$HOME/airflow
117122

123+
COPY --from=airflow-build-image --chown=${STACKABLE_USER_UID}:0 /stackable/ ${HOME}/
124+
COPY --from=airflow-build-image --chown=${STACKABLE_USER_UID}:0 /stackable/git-sync ${HOME}/git-sync
125+
126+
COPY --chown=${STACKABLE_USER_UID}:0 airflow/stackable/utils/entrypoint.sh /entrypoint.sh
127+
COPY --chown=${STACKABLE_USER_UID}:0 airflow/stackable/utils/run-airflow.sh /run-airflow.sh
128+
129+
COPY airflow/licenses /licenses
130+
118131
# Update image and install needed packages
119132
RUN <<EOF
120133
microdnf update
@@ -142,33 +155,33 @@ rm -rf /var/cache/yum
142155
# Get the correct `tini` binary for our architecture.
143156
# It is used as an init alternative in the entrypoint
144157
curl -o /usr/bin/tini "https://repo.stackable.tech/repository/packages/tini/tini-${TINI}-${TARGETARCH}"
158+
159+
# fix missing permissions
145160
chmod a+x /entrypoint.sh
146161
chmod a+x /run-airflow.sh
147162
chmod +x /usr/bin/tini
163+
EOF
148164

149-
mkdir -pv ${AIRFLOW_HOME}
150-
mkdir -pv ${AIRFLOW_HOME}/dags
151-
mkdir -pv ${AIRFLOW_HOME}/logs
165+
# ----------------------------------------
166+
# Checks
167+
# This section is to run final checks to ensure the created final images
168+
# adhere to several minimal requirements like:
169+
# - check file permissions and ownerships
170+
# ----------------------------------------
152171

153-
# All files and folders owned by root to support running as arbitrary users
154-
# This is best practice as all container users will belong to the root group (0)
155-
chown -R ${STACKABLE_USER_UID}:0 /stackable
156-
chmod -R g=u /stackable
172+
# Check that permissions and ownership in ${HOME} are set correctly
173+
# This will fail and stop the build if any mismatches are found.
174+
RUN <<EOF
175+
/bin/check-permissions-ownership.sh ${HOME} ${STACKABLE_USER_UID} 0
157176
EOF
158177

159178
# ----------------------------------------
160-
# Attention: We are changing the group of all files in /stackable directly above
161-
# If you do any file based actions (copying / creating etc.) below this comment you
162-
# absolutely need to make sure that the correct permissions are applied!
163-
# chown ${STACKABLE_USER_UID}:0
179+
# Attention: Do not perform any file based actions (copying/creating etc.) below this comment because the permissions would not be checked.
164180
# ----------------------------------------
165181

166182
USER ${STACKABLE_USER_UID}
167183
WORKDIR /stackable
168184

169-
COPY --from=airflow-build-image --chown=${STACKABLE_USER_UID}:0 /stackable/ ${HOME}/
170-
COPY --from=gitsync-image --chown=${STACKABLE_USER_UID}:0 /git-sync /stackable/git-sync
171-
172185
ENTRYPOINT ["/usr/bin/tini", "--", "/run-airflow.sh"]
173186
CMD []
174187

0 commit comments

Comments
 (0)