Skip to content

Commit 3133f8f

Browse files
ctrlaltdiljctrlaltdilj
authored
Merge pull request #4 from ctrlaltdilj/e2e-config.yaml
added e2e tests
2 parents 51993fa + 69d68f1 commit 3133f8f

File tree

7 files changed

+64
-43
lines changed

7 files changed

+64
-43
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,15 +121,24 @@ jobs:
121121
- test_multi_sessionjob.sh
122122
- test_autoscaler.sh
123123
- test_dynamic_config.sh
124+
- test_dynamic_flink_conf.sh
124125
exclude:
125126
- mode: standalone
126127
test: test_autoscaler.sh
127128
- mode: standalone
128129
test: test_dynamic_config.sh
130+
- mode: standalone
131+
test: test_dynamic_flink_conf.sh
129132
- flink-version: v1_16
130133
test: test_autoscaler.sh
131134
- flink-version: v1_16
132135
test: test_dynamic_config.sh
136+
- flink-version: v1_17
137+
test: test_dynamic_config.sh
138+
- flink-version: v1_18
139+
test: test_dynamic_config.sh
140+
- flink-version: v1_16
141+
test: test_dynamic_flink_conf.sh
133142

134143
uses: ./.github/workflows/e2e.yaml
135144
with:
@@ -156,6 +165,7 @@ jobs:
156165
- test_application_kubernetes_ha.sh
157166
- test_application_operations.sh
158167
- test_dynamic_config.sh
168+
- test_dynamic_flink_conf.sh
159169
- test_sessionjob_kubernetes_ha.sh
160170
- test_sessionjob_operations.sh
161171
- test_autoscaler.sh
@@ -169,6 +179,8 @@ jobs:
169179
test: test_autoscaler.sh
170180
- flink-version: v1_16
171181
test: test_dynamic_config.sh
182+
- flink-version: v1_16
183+
test: test_dynamic_flink_conf.sh
172184
- flink-version: v1_16
173185
test: test_flink_operator_ha.sh
174186
- flink-version: v1_16
@@ -177,6 +189,8 @@ jobs:
177189
test: test_batch_job.sh
178190
- flink-version: v1_17
179191
test: test_dynamic_config.sh
192+
- flink-version: v1_17
193+
test: test_dynamic_flink_conf.sh
180194
- flink-version: v1_17
181195
test: test_flink_operator_ha.sh
182196
- flink-version: v1_17
@@ -185,6 +199,8 @@ jobs:
185199
test: test_batch_job.sh
186200
- flink-version: v1_18
187201
test: test_dynamic_config.sh
202+
- flink-version: v1_18
203+
test: test_dynamic_flink_conf.sh
188204
- flink-version: v1_18
189205
test: test_flink_operator_ha.sh
190206
- flink-version: v1_18

‎.github/workflows/e2e.yaml‎

Lines changed: 2 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -68,51 +68,14 @@ jobs:
6868
source e2e-tests/utils.sh
6969
create_namespace ${{ inputs.namespace }}
7070
echo 'EXTRA_HELM_INSTALL_ARGS=--set "watchNamespaces={default,flink}"' >> $GITHUB_OUTPUT
71-
- name: Start the operator with legacy flink-conf.yaml
72-
run: |
73-
if [[ "${{ inputs.test }}" == "test_flink_operator_ha.sh" ]]; then
74-
sed -i "s/# kubernetes.operator.leader-election.enabled: false/kubernetes.operator.leader-election.enabled: true/" helm/flink-kubernetes-operator/conf/flink-conf.yaml
75-
sed -i "s/# kubernetes.operator.leader-election.lease-name: flink-operator-lease/kubernetes.operator.leader-election.lease-name: flink-operator-lease/" helm/flink-kubernetes-operator/conf/flink-conf.yaml
76-
sed -i "s/replicas: 1/replicas: 2/" helm/flink-kubernetes-operator/values.yaml
77-
sed -i "s/config.yaml: |+/flink-conf.yaml: |+/" helm/flink-kubernetes-operator/values.yaml
78-
fi
79-
helm --debug install flink-kubernetes-operator -n ${{ inputs.namespace }} helm/flink-kubernetes-operator --set image.repository=flink-kubernetes-operator --set image.tag=ci-latest ${{ steps.namespace-creator.outputs.EXTRA_HELM_INSTALL_ARGS }}
80-
kubectl wait --for=condition=Available --timeout=120s -n ${{ inputs.namespace }} deploy/flink-kubernetes-operator
81-
kubectl get pods -n ${{ inputs.namespace }}
82-
- name: Run Flink e2e tests using legacy flink-conf.yaml
83-
run: |
84-
FLINK_IMAGE=$(sed --regexp-extended 's/v([0-9]+)_([0-9]+)/flink:\1.\2/g' <<< ${{ inputs.flink-version }} )
85-
if [[ "${{ inputs.append-java-version }}" == "true" ]]; then
86-
FLINK_IMAGE=${FLINK_IMAGE}-java${{ inputs.java-version }}
87-
fi
88-
89-
EXAMPLES_JAR="https://repo1.maven.org/maven2/org/apache/flink/flink-examples-streaming_2.12/1.14.4/flink-examples-streaming_2.12-1.14.4.jar"
90-
if [[ ${{ inputs.flink-version }} == v2* ]]; then
91-
EXAMPLES_JAR="https://repo1.maven.org/maven2/org/apache/flink/flink-examples-streaming/2.0-preview1/flink-examples-streaming-2.0-preview1.jar"
92-
elif [[ "${{ inputs.test }}" == "test_batch_job.sh" ]]; then
93-
EXAMPLES_JAR="https://repo1.maven.org/maven2/org/apache/flink/flink-examples-streaming/1.20.1/flink-examples-streaming-1.20.1.jar"
94-
fi
95-
ESCAPED_EXAMPLES_JAR=$(printf '%s\n' "$EXAMPLES_JAR" | sed -e 's/[\/&]/\\&/g')
96-
97-
echo FLINK_IMAGE=${FLINK_IMAGE}
98-
echo EXAMPLES_JAR=${EXAMPLES_JAR}
99-
sed -i "s/image: flink:.*/image: ${FLINK_IMAGE}/" e2e-tests/data/*.yaml
100-
sed -i "s/flinkVersion: .*/flinkVersion: ${{ inputs.flink-version }}/" e2e-tests/data/*.yaml
101-
sed -i "s/mode: .*/mode: ${{ inputs.mode }}/" e2e-tests/data/*.yaml
102-
sed -i "s/STREAMING_EXAMPLES_JAR_URL/${ESCAPED_EXAMPLES_JAR}/" e2e-tests/data/*.yaml
103-
git diff HEAD
104-
echo "Running e2e-tests/$test"
105-
bash e2e-tests/${{ inputs.test }} || exit 1
106-
git reset --hard
107-
- name: Stop the operator legacy flink-conf.yaml
108-
run: |
109-
helm uninstall -n ${{ inputs.namespace }} flink-kubernetes-operator
11071
- name: Start the operator
11172
run: |
11273
if [[ "${{ inputs.test }}" == "test_flink_operator_ha.sh" ]]; then
11374
sed -i "s/# kubernetes.operator.leader-election.enabled: false/kubernetes.operator.leader-election.enabled: true/" helm/flink-kubernetes-operator/conf/flink-conf.yaml
11475
sed -i "s/# kubernetes.operator.leader-election.lease-name: flink-operator-lease/kubernetes.operator.leader-election.lease-name: flink-operator-lease/" helm/flink-kubernetes-operator/conf/flink-conf.yaml
11576
sed -i "s/replicas: 1/replicas: 2/" helm/flink-kubernetes-operator/values.yaml
77+
fi
78+
if [[ "${{ inputs.test }}" == "test_dynamic_config.sh" ]]; then
11679
sed -i "s/flink-conf.yaml: |+/config.yaml: |+/" helm/flink-kubernetes-operator/values.yaml
11780
fi
11881
helm --debug install flink-kubernetes-operator -n ${{ inputs.namespace }} helm/flink-kubernetes-operator --set image.repository=flink-kubernetes-operator --set image.tag=ci-latest ${{ steps.namespace-creator.outputs.EXTRA_HELM_INSTALL_ARGS }}

‎docs/content/docs/operations/helm.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,8 @@ The configurable parameters of the Helm chart and which default values as detail
112112
| defaultConfiguration.create | Whether to enable default configuration to create for flink-kubernetes-operator. | true |
113113
| defaultConfiguration.append | Whether to append configuration files with configs. | true |
114114
| defaultConfiguration.flink-conf.yaml | The default configuration of flink-conf.yaml. | kubernetes.operator.metrics.reporter.slf4j.factory.class: org.apache.flink.metrics.slf4j.Slf4jReporterFactory<br/>kubernetes.operator.metrics.reporter.slf4j.interval: 5 MINUTE<br/>kubernetes.operator.reconcile.interval: 15 s<br/>kubernetes.operator.observer.progress-check.interval: 5 s |
115+
| defaultConfiguration.config.yaml | The newer configuration file format for flink that will enforced in Flink 2.0. | kubernetes.operator.metrics.reporter.slf4j.factory.class: org.apache.flink.metrics.slf4j.Slf4jReporterFactory<br/>kubernetes.operator.metrics.reporter.slf4j.interval: 5 MINUTE<br/>kubernetes.operator.reconcile.interval: 15 s<br/>kubernetes.operator.observer.progress-check.interval: 5 s |
116+
115117
| defaultConfiguration.log4j-operator.properties | The default configuration of log4j-operator.properties. | |
116118
| defaultConfiguration.log4j-console.properties | The default configuration of log4j-console.properties. | |
117119
| metrics.port | The metrics port on the container for default configuration. | |

‎e2e-tests/test_dynamic_config.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ echo "Current operator pod is ${operator_pod}"
3434
create_namespace dynamic
3535

3636
kubectl config set-context --current --namespace="${operator_namespace}"
37-
patch_flink_config '{"data": {"flink-conf.yaml": "kubernetes.operator.watched.namespaces: default,flink,dynamic"}}'
37+
patch_flink_config '{"data": {"config.yaml": "kubernetes.operator.watched.namespaces: default,flink,dynamic"}}'
3838
wait_for_operator_logs "${operator_pod}" "Setting default configuration to {kubernetes.operator.watched.namespaces=default,flink,dynamic}" ${TIMEOUT} || exit 1
3939

4040
echo "Successfully run the dynamic property test"

‎e2e-tests/test_dynamic_flink_conf.sh‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/usr/bin/env bash
2+
################################################################################
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
################################################################################
19+
20+
# This script tests the operator dynamic config on watched namespace:
21+
# 1. Create a new namespace
22+
# 2. Change the watched namespaces by patching on the flink-config-override
23+
# 3. Monitor the operator log to find the watched namespace changed info
24+
SCRIPT_DIR=$(dirname "$(readlink -f "$0")")
25+
source "${SCRIPT_DIR}/utils.sh"
26+
27+
on_exit operator_cleanup_and_exit
28+
29+
TIMEOUT=360
30+
31+
operator_namespace=$(get_operator_pod_namespace)
32+
operator_pod=$(get_operator_pod_name)
33+
echo "Current operator pod is ${operator_pod}"
34+
create_namespace dynamic
35+
36+
kubectl config set-context --current --namespace="${operator_namespace}"
37+
patch_flink_config '{"data": {"flink-conf.yaml": "kubernetes.operator.watched.namespaces: default,flink,dynamic"}}'
38+
wait_for_operator_logs "${operator_pod}" "Setting default configuration to {kubernetes.operator.watched.namespaces=default,flink,dynamic}" ${TIMEOUT} || exit 1
39+
40+
echo "Successfully run the dynamic property test"

‎helm/flink-kubernetes-operator/templates/flink-operator.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -280,7 +280,7 @@ data:
280280
{{- if .Values.defaultConfiguration.append }}
281281
{{- $.Files.Get "conf/flink-conf.yaml" | nindent 4 -}}
282282
{{- end }}
283-
{{- if index (.Values.defaultConfiguration) "config.yaml" }}
283+
{{- if hasKey (.Values.defaultConfiguration) "config.yaml" }}
284284
{{- index (.Values.defaultConfiguration) "config.yaml" | nindent 4 -}}
285285
{{- end }}
286286
{{- if .Values.watchNamespaces }}
@@ -294,7 +294,7 @@ data:
294294
{{- if .Values.defaultConfiguration.append }}
295295
{{- $.Files.Get "conf/flink-conf.yaml" | nindent 4 -}}
296296
{{- end }}
297-
{{- if index (.Values.defaultConfiguration) "flink-conf.yaml" }}
297+
{{- if hasKey (.Values.defaultConfiguration) "flink-conf.yaml" }}
298298
{{- index (.Values.defaultConfiguration) "flink-conf.yaml" | nindent 4 -}}
299299
{{- end }}
300300
{{- if .Values.watchNamespaces }}

‎helm/flink-kubernetes-operator/values.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,7 @@ defaultConfiguration:
163163
# If set to false, loads just the overrides as in (2).
164164
# This option has not effect, if create is equal to false.
165165
append: true
166-
config.yaml: |+
166+
flink-conf.yaml: |+
167167
# Flink Config Overrides
168168
kubernetes.operator.metrics.reporter.slf4j.factory.class: org.apache.flink.metrics.slf4j.Slf4jReporterFactory
169169
kubernetes.operator.metrics.reporter.slf4j.interval: 5 MINUTE

0 commit comments

Comments
 (0)