Skip to content

Commit f3754f2

Browse files
SamBarkerSamBarker
committed
[FLINK-36332] reusable workflow
1 parent dced521 commit f3754f2

File tree

2 files changed

+143
-191
lines changed

2 files changed

+143
-191
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 62 additions & 191 deletions
Original file line numberDiff line numberDiff line change
@@ -73,226 +73,97 @@ jobs:
7373
cd ..
7474
e2e_smoke_test:
7575
name: Smoke test
76-
runs-on: ubuntu-latest
7776
strategy:
7877
matrix:
7978
http-client: [ "okhttp", "jdk", "jetty", "vertx" ]
80-
flink_version: [ { tag: "v1_20", image: "flink:1.20" }]
81-
mode: [ "native" ]
82-
namespace: [ "default" ]
8379
java-version: [ "11", "17", "21" ]
84-
test:
85-
- test_application_operations.sh
86-
steps:
87-
- uses: actions/checkout@v4
88-
- name: Set up JDK ${{ matrix.java-version }}
89-
uses: actions/setup-java@v4
90-
with:
91-
java-version: ${{ matrix.java-version }}
92-
distribution: 'temurin'
93-
cache: 'maven'
94-
- name: Start minikube
95-
run: |
96-
source e2e-tests/utils.sh
97-
start_minikube
98-
- name: Install cert-manager
99-
run: |
100-
kubectl get pods -A
101-
kubectl apply -f e2e-tests/cert-manager.yaml
102-
kubectl -n cert-manager wait --all=true --for=condition=Available --timeout=300s deploy
103-
- name: Build image
104-
run: |
105-
export SHELL=/bin/bash
106-
export DOCKER_BUILDKIT=1
107-
eval $(minikube -p minikube docker-env)
108-
JAVA_VERSION=${{matrix.java-version}}
109-
HTTP_CLIENT=${{ matrix.http-client }}
110-
docker build --progress=plain --no-cache -f ./Dockerfile -t flink-kubernetes-operator:ci-latest --progress plain --build-arg JAVA_VERSION="${JAVA_VERSION:-11}" --build-arg HTTP_CLIENT="${HTTP_CLIENT:-okhttp}" .
111-
docker images
112-
- name: Start the operator
113-
run: |
114-
helm --debug install flink-kubernetes-operator -n ${{ matrix.namespace }} helm/flink-kubernetes-operator --set image.repository=flink-kubernetes-operator --set image.tag=ci-latest
115-
kubectl wait --for=condition=Available --timeout=120s -n ${{ matrix.namespace }} deploy/flink-kubernetes-operator
116-
kubectl get pods -n ${{ matrix.namespace }}
117-
- name: Run Flink e2e tests
118-
run: |
119-
sed -i "s/image: flink:.*/image: ${{ matrix.flink_version.image }}/" e2e-tests/data/*.yaml
120-
sed -i "s/flinkVersion: .*/flinkVersion: ${{ matrix.flink_version.tag }}/" e2e-tests/data/*.yaml
121-
sed -i "s/mode: .*/mode: ${{ matrix.mode }}/" e2e-tests/data/*.yaml
122-
git diff HEAD
123-
echo "Running e2e-tests/$test"
124-
bash e2e-tests/${{ matrix.test }} || exit 1
125-
git reset --hard
126-
- name: Stop the operator
127-
run: |
128-
helm uninstall -n ${{ matrix.namespace }} flink-kubernetes-operator
129-
- name: Stop minikube
130-
run: |
131-
source e2e-tests/utils.sh
132-
stop_minikube
80+
uses: ./.github/workflows/e2e.yaml
81+
with:
82+
java-version: ${{ matrix.java-version }}
83+
flink-version: "v1_20"
84+
http-client: ${{ matrix.http-client }}
85+
test: test_application_operations.sh
13386
e2e_namespace_tests:
134-
name: Namespace aware tests
135-
runs-on: ubuntu-latest
87+
name: Alternative namespace tests
13688
needs: e2e_smoke_test
13789
strategy:
13890
matrix:
139-
# Define a single matrix entry for the tag image mapping https://github.com/orgs/community/discussions/24981#discussioncomment-3246068
140-
flink_version: [
141-
{ tag: "v1_20", image: "flink:1.20" },
142-
{ tag: "v1_18", image: "flink:1.18" },
143-
{ tag: "v1_19", image: "flink:1.19" },
144-
{ tag: "v1_17", image: "flink:1.17" },
145-
{ tag: "v1_16", image: "flink:1.16" } ]
146-
mode: [ "native", "standalone" ]
147-
namespace: [ "flink" ]
148-
java-version: [ "11" ]
91+
flink-version:
92+
- "v1_20"
93+
- "v1_18"
94+
- "v1_19"
95+
- "v1_17"
96+
- "v1_16"
97+
mode:
98+
- "native"
99+
- "standalone"
149100
test:
150101
- test_application_operations.sh
151102
- test_multi_sessionjob.sh
152103
- test_autoscaler.sh
153104
- test_dynamic_config.sh
154105
excludes:
155-
- { mode: standalone, test: test_autoscaler.sh }
156-
- { mode: standalone, test: test_dynamic_config.sh }
157-
- { flink_version: {tag: "v1_17", image: "flink:1.17"}, test: test_dynamic_config.sh }
158-
- { flink_version: {tag: "v1_16", image: "flink:1.16"}, test: test_autoscaler.sh }
159-
- { flink_version: {tag: "v1_16", image: "flink:1.16"}, test: test_dynamic_config.sh }
160-
steps:
161-
- uses: actions/checkout@v4
162-
- name: Set up JDK ${{ matrix.java-version }}
163-
uses: actions/setup-java@v4
164-
with:
165-
java-version: ${{ matrix.java-version }}
166-
distribution: 'temurin'
167-
cache: 'maven'
168-
- name: Start minikube
169-
run: |
170-
source e2e-tests/utils.sh
171-
start_minikube
172-
- name: Install cert-manager
173-
run: |
174-
kubectl get pods -A
175-
kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.8.2/cert-manager.yaml
176-
kubectl -n cert-manager wait --all=true --for=condition=Available --timeout=300s deploy
177-
- name: Build image
178-
run: |
179-
export SHELL=/bin/bash
180-
export DOCKER_BUILDKIT=1
181-
eval $(minikube -p minikube docker-env)
182-
docker build --progress=plain --no-cache -f ./Dockerfile -t flink-kubernetes-operator:ci-latest --progress plain .
183-
docker images
184-
- name: Start the operator
185-
run: |
186-
source e2e-tests/utils.sh
187-
create_namespace flink
188-
helm --debug install flink-kubernetes-operator -n ${{ matrix.namespace }} helm/flink-kubernetes-operator --set image.repository=flink-kubernetes-operator --set image.tag=ci-latest --create-namespace --set "watchNamespaces={default,flink}"
189-
kubectl wait --for=condition=Available --timeout=120s -n ${{ matrix.namespace }} deploy/flink-kubernetes-operator
190-
kubectl get pods -n ${{ matrix.namespace }}
191-
- name: Run Flink e2e tests
192-
run: |
193-
sed -i "s/image: flink:.*/image: ${{ matrix.flink_version.image }}/" e2e-tests/data/*.yaml
194-
sed -i "s/flinkVersion: .*/flinkVersion: ${{ matrix.flink_version.tag }}/" e2e-tests/data/*.yaml
195-
sed -i "s/mode: .*/mode: ${{ matrix.mode }}/" e2e-tests/data/*.yaml
196-
git diff HEAD
197-
echo "Running e2e-tests/$test"
198-
bash e2e-tests/${{ matrix.test }} || exit 1
199-
git reset --hard
200-
- name: Stop the operator
201-
run: |
202-
helm uninstall -n ${{ matrix.namespace }} flink-kubernetes-operator
203-
- name: Stop minikube
204-
run: |
205-
source e2e-tests/utils.sh
206-
stop_minikube
106+
- mode: standalone
107+
test: test_autoscaler.sh
108+
- mode: standalone
109+
test: test_dynamic_config.sh
110+
uses: ./.github/workflows/e2e.yaml
111+
with:
112+
java-version: 11
113+
flink-version: ${{ matrix.flink-version }}
114+
test: ${{ matrix.test }}
115+
namespace: "flink"
116+
mode: ${{ matrix.mode }}
207117
e2e_ci:
208-
name: e2e_ci
209118
needs: e2e_smoke_test
210-
runs-on: ubuntu-latest
211119
strategy:
212120
matrix:
213-
# Define a single matrix entry for the tag image mapping https://github.com/orgs/community/discussions/24981#discussioncomment-3246068
214-
flink_version: [
215-
{ tag: "v1_20", image: "flink:1.20" },
216-
{ tag: "v1_18", image: "flink:1.18" },
217-
{ tag: "v1_19", image: "flink:1.19" },
218-
{ tag: "v1_17", image: "flink:1.17" },
219-
{ tag: "v1_16", image: "flink:1.16" } ]
220-
namespace: [ "default" ]
221-
mode: [ "native", "standalone" ]
222-
java-version: [ 11 ]
121+
flink-version:
122+
- "v1_20"
123+
- "v1_18"
124+
- "v1_19"
125+
- "v1_17"
126+
- "v1_16"
127+
mode:
128+
- "native"
129+
- "standalone"
223130
test:
224131
- test_application_kubernetes_ha.sh
132+
- test_application_operations.sh
133+
- test_dynamic_config.sh
225134
- test_sessionjob_kubernetes_ha.sh
226135
- test_sessionjob_operations.sh
136+
- test_multi_sessionjob.sh
137+
- test_autoscaler.sh
227138
- test_flink_operator_ha.sh
228139
- test_snapshot.sh
229-
exclude:
230-
- mode: standalone
231-
test: test_snapshot.sh
232-
- flink_version: { tag: "v1_16", image: "flink:1.16" }
140+
excludes:
141+
- version: v1_16
142+
test: test_autoscaler.sh
143+
- version: v1_16
144+
test: test_dynamic_config.sh
145+
- version: v1_16
233146
test: test_flink_operator_ha.sh
234-
- flink_version: { tag: "v1_16", image: "flink:1.16" }
147+
- version: v1_16
235148
test: test_snapshot.sh
236-
- flink_version: { tag: "v1_17", image: "flink:1.17" }
149+
- version: v1_17
150+
test: test_dynamic_config.sh
151+
- version: v1_17
237152
test: test_flink_operator_ha.sh
238-
- flink_version: { tag: "v1_17", image: "flink:1.17" }
153+
- version: v1_17
239154
test: test_snapshot.sh
240-
- flink_version: { tag: "v1_18", image: "flink:1.18" }
155+
- version: v1_18
241156
test: test_dynamic_config.sh
242-
- flink_version: { tag: "v1_18", image: "flink:1.18" }
157+
- version: v1_18
243158
test: test_flink_operator_ha.sh
244-
- flink_version: { tag: "v1_18", image: "flink:1.18" }
159+
- version: v1_18
245160
test: test_snapshot.sh
246-
- flink_version: { tag: "v1_19", image: "flink:1.19" }
161+
- version: v1_19
247162
test: test_snapshot.sh
248-
steps:
249-
- uses: actions/checkout@v4
250-
- name: Set up JDK ${{ matrix.java-version }}
251-
uses: actions/setup-java@v4
252-
with:
253-
java-version: ${{ matrix.java-version }}
254-
distribution: 'temurin'
255-
cache: 'maven'
256-
- name: Start minikube
257-
run: |
258-
source e2e-tests/utils.sh
259-
start_minikube
260-
- name: Install cert-manager
261-
run: |
262-
kubectl get pods -A
263-
kubectl apply -f e2e-tests/cert-manager.yaml
264-
kubectl -n cert-manager wait --all=true --for=condition=Available --timeout=300s deploy
265-
- name: Build image
266-
run: |
267-
export SHELL=/bin/bash
268-
export DOCKER_BUILDKIT=1
269-
eval $(minikube -p minikube docker-env)
270-
JAVA_VERSION=${{ matrix.java-version }}
271-
docker build --progress=plain --no-cache -f ./Dockerfile -t flink-kubernetes-operator:ci-latest --progress plain --build-arg JAVA_VERSION="${JAVA_VERSION:-11}" .
272-
docker images
273-
- name: Start the operator
274-
run: |
275-
if [[ "${{ matrix.test }}" == "test_flink_operator_ha.sh" ]]; then
276-
sed -i "s/# kubernetes.operator.leader-election.enabled: false/kubernetes.operator.leader-election.enabled: true/" helm/flink-kubernetes-operator/conf/flink-conf.yaml
277-
sed -i "s/# kubernetes.operator.leader-election.lease-name: flink-operator-lease/kubernetes.operator.leader-election.lease-name: flink-operator-lease/" helm/flink-kubernetes-operator/conf/flink-conf.yaml
278-
sed -i "s/replicas: 1/replicas: 2/" helm/flink-kubernetes-operator/values.yaml
279-
fi
280-
helm --debug install flink-kubernetes-operator -n ${{ matrix.namespace }} helm/flink-kubernetes-operator --set image.repository=flink-kubernetes-operator --set image.tag=ci-latest
281-
kubectl wait --for=condition=Available --timeout=120s -n ${{ matrix.namespace }} deploy/flink-kubernetes-operator
282-
kubectl get pods -n ${{ matrix.namespace }}
283-
- name: Run Flink e2e tests
284-
run: |
285-
sed -i "s/image: flink:.*/image: ${{ matrix.flink_version.image }}/" e2e-tests/data/*.yaml
286-
sed -i "s/flinkVersion: .*/flinkVersion: ${{ matrix.flink_version.tag }}/" e2e-tests/data/*.yaml
287-
sed -i "s/mode: .*/mode: ${{ matrix.mode }}/" e2e-tests/data/*.yaml
288-
git diff HEAD
289-
echo "Running e2e-tests/$test"
290-
bash e2e-tests/${{ matrix.test }} || exit 1
291-
git reset --hard
292-
- name: Stop the operator
293-
run: |
294-
helm uninstall -n ${{ matrix.namespace }} flink-kubernetes-operator
295-
- name: Stop minikube
296-
run: |
297-
source e2e-tests/utils.sh
298-
stop_minikube
163+
uses: ./.github/workflows/e2e.yaml
164+
with:
165+
java-version: 11
166+
flink-version: ${{ matrix.flink-version }}
167+
test: ${{ matrix.test }}
168+
namespace: "flink"
169+
mode: ${{ matrix.mode }}

‎.github/workflows/e2e.yaml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
name: E2E test workflow
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
java-version:
7+
required: true
8+
type: string
9+
flink-version:
10+
required: true
11+
type: string
12+
http-client:
13+
required: false
14+
type: string
15+
default: "okhttp"
16+
namespace:
17+
required: false
18+
type: string
19+
default: "default"
20+
mode:
21+
required: false
22+
type: string
23+
default: "native"
24+
test:
25+
required: true
26+
type: string
27+
28+
jobs:
29+
e2e_test:
30+
name: ${{ inputs.test }}
31+
runs-on: ubuntu-latest
32+
steps:
33+
- uses: actions/checkout@v4
34+
- name: Set up JDK ${{ inputs.java-version }}
35+
uses: actions/setup-java@v4
36+
with:
37+
java-version: ${{ inputs.java-version }}
38+
distribution: 'temurin'
39+
cache: 'maven'
40+
- name: Start minikube
41+
run: |
42+
source e2e-tests/utils.sh
43+
start_minikube
44+
- name: Install cert-manager
45+
run: |
46+
kubectl get pods -A
47+
kubectl apply -f e2e-tests/cert-manager.yaml
48+
kubectl -n cert-manager wait --all=true --for=condition=Available --timeout=300s deploy
49+
- name: Build image
50+
run: |
51+
export SHELL=/bin/bash
52+
export DOCKER_BUILDKIT=1
53+
eval $(minikube -p minikube docker-env)
54+
JAVA_VERSION=${{ inputs.java-version }}
55+
HTTP_CLIENT=${{ inputs.http-client }}
56+
docker build --progress=plain --no-cache -f ./Dockerfile -t flink-kubernetes-operator:ci-latest --progress plain --build-arg JAVA_VERSION="${JAVA_VERSION:-11}" --build-arg HTTP_CLIENT="${HTTP_CLIENT:-okhttp}" .
57+
docker images
58+
- name: Start the operator
59+
run: |
60+
helm --debug install flink-kubernetes-operator -n ${{ inputs.namespace }} helm/flink-kubernetes-operator --set image.repository=flink-kubernetes-operator --set image.tag=ci-latest
61+
kubectl wait --for=condition=Available --timeout=120s -n ${{ inputs.namespace }} deploy/flink-kubernetes-operator
62+
kubectl get pods -n ${{ inputs.namespace }}
63+
- name: Run Flink e2e tests
64+
run: |
65+
FLINK_IMAGE=$(sed --regexp-extended 's/v([0-9]+)_([0-9]+)/flink:\1.\2/g' <<< ${{ inputs.flink-version }} )
66+
echo FLINK_IMAGE=${FLINK_IMAGE}
67+
sed -i "s/image: flink:.*/image: ${FLINK_IMAGE}/" e2e-tests/data/*.yaml
68+
sed -i "s/flinkVersion: .*/flinkVersion: ${{ inputs.flink-version }}/" e2e-tests/data/*.yaml
69+
sed -i "s/mode: .*/mode: ${{ inputs.mode }}/" e2e-tests/data/*.yaml
70+
git diff HEAD
71+
echo "Running e2e-tests/$test"
72+
bash e2e-tests/${{ inputs.test }} || exit 1
73+
git reset --hard
74+
- name: Stop the operator
75+
run: |
76+
helm uninstall -n ${{ inputs.namespace }} flink-kubernetes-operator
77+
- name: Stop minikube
78+
run: |
79+
source e2e-tests/utils.sh
80+
stop_minikube
81+

0 commit comments

Comments
 (0)