Skip to content

Commit e5bb8ce

Browse files
SamBarkerSamBarker
committed
[FLINK-36332] reusable workflow
1 parent 2e9039e commit e5bb8ce

File tree

1 file changed

+171
-213
lines changed

1 file changed

+171
-213
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 171 additions & 213 deletions
Original file line numberDiff line numberDiff line change
@@ -77,222 +77,180 @@ jobs:
7777
strategy:
7878
matrix:
7979
http-client: [ "okhttp", "jdk", "jetty", "vertx" ]
80-
flink_version: [ { tag: "v1_20", image: "flink:1.20" }]
81-
mode: [ "native" ]
82-
namespace: [ "default" ]
8380
java-version: [ "11", "17", "21" ]
8481
test:
8582
- test_application_operations.sh
8683
steps:
87-
- uses: actions/checkout@v4
88-
- name: Set up JDK ${{ matrix.java-version }}
89-
uses: actions/setup-java@v4
84+
- uses: ./.github/workflows/e2e.yaml
9085
with:
9186
java-version: ${{ matrix.java-version }}
92-
distribution: 'temurin'
93-
cache: 'maven'
94-
- name: Start minikube
95-
run: |
96-
source e2e-tests/utils.sh
97-
start_minikube
98-
- name: Install cert-manager
99-
run: |
100-
kubectl get pods -A
101-
kubectl apply -f e2e-tests/cert-manager.yaml
102-
kubectl -n cert-manager wait --all=true --for=condition=Available --timeout=300s deploy
103-
- name: Build image
104-
run: |
105-
export SHELL=/bin/bash
106-
export DOCKER_BUILDKIT=1
107-
eval $(minikube -p minikube docker-env)
108-
JAVA_VERSION=${{matrix.java-version}}
109-
HTTP_CLIENT=${{ matrix.http-client }}
110-
docker build --progress=plain --no-cache -f ./Dockerfile -t flink-kubernetes-operator:ci-latest --progress plain --build-arg JAVA_VERSION="${JAVA_VERSION:-11}" --build-arg HTTP_CLIENT="${HTTP_CLIENT:-okhttp}" .
111-
docker images
112-
- name: Start the operator
113-
run: |
114-
helm --debug install flink-kubernetes-operator -n ${{ matrix.namespace }} helm/flink-kubernetes-operator --set image.repository=flink-kubernetes-operator --set image.tag=ci-latest
115-
kubectl wait --for=condition=Available --timeout=120s -n ${{ matrix.namespace }} deploy/flink-kubernetes-operator
116-
kubectl get pods -n ${{ matrix.namespace }}
117-
- name: Run Flink e2e tests
118-
run: |
119-
sed -i "s/image: flink:.*/image: ${{ matrix.flink_version.image }}/" e2e-tests/data/*.yaml
120-
sed -i "s/flinkVersion: .*/flinkVersion: ${{ matrix.flink_version.tag }}/" e2e-tests/data/*.yaml
121-
sed -i "s/mode: .*/mode: ${{ matrix.mode }}/" e2e-tests/data/*.yaml
122-
git diff HEAD
123-
echo "Running e2e-tests/$test"
124-
bash e2e-tests/${{ matrix.test }} || exit 1
125-
git reset --hard
126-
- name: Stop the operator
127-
run: |
128-
helm uninstall -n ${{ matrix.namespace }} flink-kubernetes-operator
129-
- name: Stop minikube
130-
run: |
131-
source e2e-tests/utils.sh
132-
stop_minikube
133-
e2e_namespace_tests:
134-
name: Namespace aware tests
135-
runs-on: ubuntu-latest
136-
needs: e2e_smoke_test
137-
strategy:
138-
matrix:
139-
# Define a single matrix entry for the tag image mapping https://github.com/orgs/community/discussions/24981#discussioncomment-3246068
140-
flink_version: [
141-
{ tag: "v1_20", image: "flink:1.20" },
142-
{ tag: "v1_18", image: "flink:1.18" },
143-
{ tag: "v1_19", image: "flink:1.19" },
144-
{ tag: "v1_17", image: "flink:1.17" },
145-
{ tag: "v1_16", image: "flink:1.16" } ]
146-
mode: [ "native", "standalone" ]
147-
namespace: [ "flink" ]
148-
java-version: [ "11" ]
149-
test:
150-
- test_application_operations.sh
151-
- test_multi_sessionjob.sh
152-
- test_autoscaler.sh
153-
- test_dynamic_config.sh
154-
excludes:
155-
- { mode: standalone, test: test_autoscaler.sh }
156-
- { mode: standalone, test: test_dynamic_config.sh }
157-
- { flink_version: {tag: "v1_17", image: "flink:1.17"}, test: test_dynamic_config.sh }
158-
- { flink_version: {tag: "v1_16", image: "flink:1.16"}, test: test_autoscaler.sh }
159-
- { flink_version: {tag: "v1_16", image: "flink:1.16"}, test: test_dynamic_config.sh }
160-
steps:
161-
- uses: actions/checkout@v4
162-
- name: Set up JDK ${{ matrix.java-version }}
163-
uses: actions/setup-java@v4
164-
with:
165-
java-version: ${{ matrix.java-version }}
166-
distribution: 'temurin'
167-
cache: 'maven'
168-
- name: Start minikube
169-
run: |
170-
source e2e-tests/utils.sh
171-
start_minikube
172-
- name: Install cert-manager
173-
run: |
174-
kubectl get pods -A
175-
kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.8.2/cert-manager.yaml
176-
kubectl -n cert-manager wait --all=true --for=condition=Available --timeout=300s deploy
177-
- name: Build image
178-
run: |
179-
export SHELL=/bin/bash
180-
export DOCKER_BUILDKIT=1
181-
eval $(minikube -p minikube docker-env)
182-
docker build --progress=plain --no-cache -f ./Dockerfile -t flink-kubernetes-operator:ci-latest --progress plain .
183-
docker images
184-
- name: Start the operator
185-
run: |
186-
source e2e-tests/utils.sh
187-
create_namespace flink
188-
helm --debug install flink-kubernetes-operator -n ${{ matrix.namespace }} helm/flink-kubernetes-operator --set image.repository=flink-kubernetes-operator --set image.tag=ci-latest --create-namespace --set "watchNamespaces={default,flink}"
189-
kubectl wait --for=condition=Available --timeout=120s -n ${{ matrix.namespace }} deploy/flink-kubernetes-operator
190-
kubectl get pods -n ${{ matrix.namespace }}
191-
- name: Run Flink e2e tests
192-
run: |
193-
sed -i "s/image: flink:.*/image: ${{ matrix.flink_version.image }}/" e2e-tests/data/*.yaml
194-
sed -i "s/flinkVersion: .*/flinkVersion: ${{ matrix.flink_version.tag }}/" e2e-tests/data/*.yaml
195-
sed -i "s/mode: .*/mode: ${{ matrix.mode }}/" e2e-tests/data/*.yaml
196-
git diff HEAD
197-
echo "Running e2e-tests/$test"
198-
bash e2e-tests/${{ matrix.test }} || exit 1
199-
git reset --hard
200-
- name: Stop the operator
201-
run: |
202-
helm uninstall -n ${{ matrix.namespace }} flink-kubernetes-operator
203-
- name: Stop minikube
204-
run: |
205-
source e2e-tests/utils.sh
206-
stop_minikube
207-
e2e_ci:
208-
name: e2e_ci
209-
needs: e2e_smoke_test
210-
runs-on: ubuntu-latest
211-
strategy:
212-
matrix:
213-
# Define a single matrix entry for the tag image mapping https://github.com/orgs/community/discussions/24981#discussioncomment-3246068
214-
flink_version: [
215-
{ tag: "v1_20", image: "flink:1.20" },
216-
{ tag: "v1_18", image: "flink:1.18" },
217-
{ tag: "v1_19", image: "flink:1.19" },
218-
{ tag: "v1_17", image: "flink:1.17" },
219-
{ tag: "v1_16", image: "flink:1.16" } ]
220-
namespace: [ "default" ]
221-
mode: [ "native", "standalone" ]
222-
java-version: [ 11 ]
223-
test:
224-
- test_application_kubernetes_ha.sh
225-
- test_sessionjob_kubernetes_ha.sh
226-
- test_sessionjob_operations.sh
227-
- test_flink_operator_ha.sh
228-
- test_snapshot.sh
229-
exclude:
230-
- mode: standalone
231-
test: test_snapshot.sh
232-
- flink_version: { tag: "v1_16", image: "flink:1.16" }
233-
test: test_flink_operator_ha.sh
234-
- flink_version: { tag: "v1_16", image: "flink:1.16" }
235-
test: test_snapshot.sh
236-
- flink_version: { tag: "v1_17", image: "flink:1.17" }
237-
test: test_flink_operator_ha.sh
238-
- flink_version: { tag: "v1_17", image: "flink:1.17" }
239-
test: test_snapshot.sh
240-
- flink_version: { tag: "v1_18", image: "flink:1.18" }
241-
test: test_dynamic_config.sh
242-
- flink_version: { tag: "v1_18", image: "flink:1.18" }
243-
test: test_flink_operator_ha.sh
244-
- flink_version: { tag: "v1_18", image: "flink:1.18" }
245-
test: test_snapshot.sh
246-
- flink_version: { tag: "v1_19", image: "flink:1.19" }
247-
test: test_snapshot.sh
248-
steps:
249-
- uses: actions/checkout@v4
250-
- name: Set up JDK ${{ matrix.java-version }}
251-
uses: actions/setup-java@v4
252-
with:
253-
java-version: ${{ matrix.java-version }}
254-
distribution: 'temurin'
255-
cache: 'maven'
256-
- name: Start minikube
257-
run: |
258-
source e2e-tests/utils.sh
259-
start_minikube
260-
- name: Install cert-manager
261-
run: |
262-
kubectl get pods -A
263-
kubectl apply -f e2e-tests/cert-manager.yaml
264-
kubectl -n cert-manager wait --all=true --for=condition=Available --timeout=300s deploy
265-
- name: Build image
266-
run: |
267-
export SHELL=/bin/bash
268-
export DOCKER_BUILDKIT=1
269-
eval $(minikube -p minikube docker-env)
270-
JAVA_VERSION=${{ matrix.java-version }}
271-
docker build --progress=plain --no-cache -f ./Dockerfile -t flink-kubernetes-operator:ci-latest --progress plain --build-arg JAVA_VERSION="${JAVA_VERSION:-11}" .
272-
docker images
273-
- name: Start the operator
274-
run: |
275-
if [[ "${{ matrix.test }}" == "test_flink_operator_ha.sh" ]]; then
276-
sed -i "s/# kubernetes.operator.leader-election.enabled: false/kubernetes.operator.leader-election.enabled: true/" helm/flink-kubernetes-operator/conf/flink-conf.yaml
277-
sed -i "s/# kubernetes.operator.leader-election.lease-name: flink-operator-lease/kubernetes.operator.leader-election.lease-name: flink-operator-lease/" helm/flink-kubernetes-operator/conf/flink-conf.yaml
278-
sed -i "s/replicas: 1/replicas: 2/" helm/flink-kubernetes-operator/values.yaml
279-
fi
280-
helm --debug install flink-kubernetes-operator -n ${{ matrix.namespace }} helm/flink-kubernetes-operator --set image.repository=flink-kubernetes-operator --set image.tag=ci-latest
281-
kubectl wait --for=condition=Available --timeout=120s -n ${{ matrix.namespace }} deploy/flink-kubernetes-operator
282-
kubectl get pods -n ${{ matrix.namespace }}
283-
- name: Run Flink e2e tests
284-
run: |
285-
sed -i "s/image: flink:.*/image: ${{ matrix.flink_version.image }}/" e2e-tests/data/*.yaml
286-
sed -i "s/flinkVersion: .*/flinkVersion: ${{ matrix.flink_version.tag }}/" e2e-tests/data/*.yaml
287-
sed -i "s/mode: .*/mode: ${{ matrix.mode }}/" e2e-tests/data/*.yaml
288-
git diff HEAD
289-
echo "Running e2e-tests/$test"
290-
bash e2e-tests/${{ matrix.test }} || exit 1
291-
git reset --hard
292-
- name: Stop the operator
293-
run: |
294-
helm uninstall -n ${{ matrix.namespace }} flink-kubernetes-operator
295-
- name: Stop minikube
296-
run: |
297-
source e2e-tests/utils.sh
298-
stop_minikube
87+
flink-version: "v1_20"
88+
flink-image: "flink:1.20"
89+
http-client: ${{ matrix.http-client }}
90+
91+
# e2e_namespace_tests:
92+
# name: Namespace aware tests
93+
# runs-on: ubuntu-latest
94+
# needs: e2e_smoke_test
95+
# strategy:
96+
# matrix:
97+
# # Define a single matrix entry for the tag image mapping https://github.com/orgs/community/discussions/24981#discussioncomment-3246068
98+
# flink_version: [
99+
# { tag: "v1_20", image: "flink:1.20" },
100+
# { tag: "v1_18", image: "flink:1.18" },
101+
# { tag: "v1_19", image: "flink:1.19" },
102+
# { tag: "v1_17", image: "flink:1.17" },
103+
# { tag: "v1_16", image: "flink:1.16" } ]
104+
# mode: [ "native", "standalone" ]
105+
# namespace: [ "flink" ]
106+
# java-version: [ "11" ]
107+
# test:
108+
# - test_application_operations.sh
109+
# - test_multi_sessionjob.sh
110+
# - test_autoscaler.sh
111+
# - test_dynamic_config.sh
112+
# excludes:
113+
# - { mode: standalone, test: test_autoscaler.sh }
114+
# - { mode: standalone, test: test_dynamic_config.sh }
115+
# - { flink_version: [{tag: "v1_17", image: "flink:1.17"}], test: test_dynamic_config.sh }
116+
# - { flink_version: [{tag: "v1_16", image: "flink:1.16"}], test: test_autoscaler.sh }
117+
# - { flink_version: [{tag: "v1_16", image: "flink:1.16"}], test: test_dynamic_config.sh }
118+
# steps:
119+
# - uses: actions/checkout@v4
120+
# - name: Set up JDK ${{ matrix.java-version }}
121+
# uses: actions/setup-java@v4
122+
# with:
123+
# java-version: ${{ matrix.java-version }}
124+
# distribution: 'temurin'
125+
# cache: 'maven'
126+
# - name: Start minikube
127+
# run: |
128+
# source e2e-tests/utils.sh
129+
# start_minikube
130+
# - name: Install cert-manager
131+
# run: |
132+
# kubectl get pods -A
133+
# kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.8.2/cert-manager.yaml
134+
# kubectl -n cert-manager wait --all=true --for=condition=Available --timeout=300s deploy
135+
# - name: Build image
136+
# run: |
137+
# export SHELL=/bin/bash
138+
# export DOCKER_BUILDKIT=1
139+
# eval $(minikube -p minikube docker-env)
140+
# docker build --progress=plain --no-cache -f ./Dockerfile -t flink-kubernetes-operator:ci-latest --progress plain .
141+
# docker images
142+
# - name: Start the operator
143+
# run: |
144+
# source e2e-tests/utils.sh
145+
# create_namespace flink
146+
# helm --debug install flink-kubernetes-operator -n ${{ matrix.namespace }} helm/flink-kubernetes-operator --set image.repository=flink-kubernetes-operator --set image.tag=ci-latest --create-namespace --set "watchNamespaces={default,flink}"
147+
# kubectl wait --for=condition=Available --timeout=120s -n ${{ matrix.namespace }} deploy/flink-kubernetes-operator
148+
# kubectl get pods -n ${{ matrix.namespace }}
149+
# - name: Run Flink e2e tests
150+
# run: |
151+
# sed -i "s/image: flink:.*/image: ${{ matrix.flink_version.image }}/" e2e-tests/data/*.yaml
152+
# sed -i "s/flinkVersion: .*/flinkVersion: ${{ matrix.flink_version.tag }}/" e2e-tests/data/*.yaml
153+
# sed -i "s/mode: .*/mode: ${{ matrix.mode }}/" e2e-tests/data/*.yaml
154+
# git diff HEAD
155+
# echo "Running e2e-tests/$test"
156+
# bash e2e-tests/${{ matrix.test }} || exit 1
157+
# git reset --hard
158+
# - name: Stop the operator
159+
# run: |
160+
# helm uninstall -n ${{ matrix.namespace }} flink-kubernetes-operator
161+
# - name: Stop minikube
162+
# run: |
163+
# source e2e-tests/utils.sh
164+
# stop_minikube
165+
# e2e_ci:
166+
# name: e2e_ci
167+
# needs: e2e_smoke_test
168+
# runs-on: ubuntu-latest
169+
# strategy:
170+
# matrix:
171+
# # Define a single matrix entry for the tag image mapping https://github.com/orgs/community/discussions/24981#discussioncomment-3246068
172+
# flink_version: [
173+
# { tag: "v1_20", image: "flink:1.20" },
174+
# { tag: "v1_18", image: "flink:1.18" },
175+
# { tag: "v1_19", image: "flink:1.19" },
176+
# { tag: "v1_17", image: "flink:1.17" },
177+
# { tag: "v1_16", image: "flink:1.16" } ]
178+
# namespace: [ "default" ]
179+
# mode: [ "native", "standalone" ]
180+
# java-version: [ 11 ]
181+
# test:
182+
# - test_application_kubernetes_ha.sh
183+
# - test_sessionjob_kubernetes_ha.sh
184+
# - test_sessionjob_operations.sh
185+
# - test_flink_operator_ha.sh
186+
# - test_snapshot.sh
187+
# exclude:
188+
# - mode: standalone
189+
# test: test_snapshot.sh
190+
# - flink_version: { tag: "v1_16", image: "flink:1.16" }
191+
# test: test_flink_operator_ha.sh
192+
# - flink_version: { tag: "v1_16", image: "flink:1.16" }
193+
# test: test_snapshot.sh
194+
# - flink_version: { tag: "v1_17", image: "flink:1.17" }
195+
# test: test_flink_operator_ha.sh
196+
# - flink_version: { tag: "v1_17", image: "flink:1.17" }
197+
# test: test_snapshot.sh
198+
# - flink_version: { tag: "v1_18", image: "flink:1.18" }
199+
# test: test_dynamic_config.sh
200+
# - flink_version: { tag: "v1_18", image: "flink:1.18" }
201+
# test: test_flink_operator_ha.sh
202+
# - flink_version: { tag: "v1_18", image: "flink:1.18" }
203+
# test: test_snapshot.sh
204+
# - flink_version: { tag: "v1_19", image: "flink:1.19" }
205+
# test: test_snapshot.sh
206+
# steps:
207+
# - uses: actions/checkout@v4
208+
# - name: Set up JDK ${{ matrix.java-version }}
209+
# uses: actions/setup-java@v4
210+
# with:
211+
# java-version: ${{ matrix.java-version }}
212+
# distribution: 'temurin'
213+
# cache: 'maven'
214+
# - name: Start minikube
215+
# run: |
216+
# source e2e-tests/utils.sh
217+
# start_minikube
218+
# - name: Install cert-manager
219+
# run: |
220+
# kubectl get pods -A
221+
# kubectl apply -f e2e-tests/cert-manager.yaml
222+
# kubectl -n cert-manager wait --all=true --for=condition=Available --timeout=300s deploy
223+
# - name: Build image
224+
# run: |
225+
# export SHELL=/bin/bash
226+
# export DOCKER_BUILDKIT=1
227+
# eval $(minikube -p minikube docker-env)
228+
# java-version=${{ matrix.java-version }}
229+
# docker build --progress=plain --no-cache -f ./Dockerfile -t flink-kubernetes-operator:ci-latest --progress plain --build-arg java-version="${java-version:-11}" .
230+
# docker images
231+
# - name: Start the operator
232+
# run: |
233+
# if [[ "${{ matrix.test }}" == "test_flink_operator_ha.sh" ]]; then
234+
# sed -i "s/# kubernetes.operator.leader-election.enabled: false/kubernetes.operator.leader-election.enabled: true/" helm/flink-kubernetes-operator/conf/flink-conf.yaml
235+
# sed -i "s/# kubernetes.operator.leader-election.lease-name: flink-operator-lease/kubernetes.operator.leader-election.lease-name: flink-operator-lease/" helm/flink-kubernetes-operator/conf/flink-conf.yaml
236+
# sed -i "s/replicas: 1/replicas: 2/" helm/flink-kubernetes-operator/values.yaml
237+
# fi
238+
# helm --debug install flink-kubernetes-operator -n ${{ matrix.namespace }} helm/flink-kubernetes-operator --set image.repository=flink-kubernetes-operator --set image.tag=ci-latest
239+
# kubectl wait --for=condition=Available --timeout=120s -n ${{ matrix.namespace }} deploy/flink-kubernetes-operator
240+
# kubectl get pods -n ${{ matrix.namespace }}
241+
# - name: Run Flink e2e tests
242+
# run: |
243+
# sed -i "s/image: flink:.*/image: ${{ matrix.flink_version.image }}/" e2e-tests/data/*.yaml
244+
# sed -i "s/flinkVersion: .*/flinkVersion: ${{ matrix.flink_version.tag }}/" e2e-tests/data/*.yaml
245+
# sed -i "s/mode: .*/mode: ${{ matrix.mode }}/" e2e-tests/data/*.yaml
246+
# git diff HEAD
247+
# echo "Running e2e-tests/$test"
248+
# bash e2e-tests/${{ matrix.test }} || exit 1
249+
# git reset --hard
250+
# - name: Stop the operator
251+
# run: |
252+
# helm uninstall -n ${{ matrix.namespace }} flink-kubernetes-operator
253+
# - name: Stop minikube
254+
# run: |
255+
# source e2e-tests/utils.sh
256+
# stop_minikube

0 commit comments

Comments
 (0)