Skip to content

Commit 2bf49a2

Browse files
ctrlaltdiljctrlaltdilj
authored
[FLINK-35744] Support flink 1.19 config.yaml format
1 parent 44b1ff6 commit 2bf49a2

File tree

6 files changed

+82
-2
lines changed

6 files changed

+82
-2
lines changed

‎.github/workflows/ci.yml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,15 +121,24 @@ jobs:
121121
- test_multi_sessionjob.sh
122122
- test_autoscaler.sh
123123
- test_dynamic_config.sh
124+
- test_dynamic_flink_conf.sh
124125
exclude:
125126
- mode: standalone
126127
test: test_autoscaler.sh
127128
- mode: standalone
128129
test: test_dynamic_config.sh
130+
- mode: standalone
131+
test: test_dynamic_flink_conf.sh
129132
- flink-version: v1_16
130133
test: test_autoscaler.sh
131134
- flink-version: v1_16
132135
test: test_dynamic_config.sh
136+
- flink-version: v1_17
137+
test: test_dynamic_config.sh
138+
- flink-version: v1_18
139+
test: test_dynamic_config.sh
140+
- flink-version: v1_16
141+
test: test_dynamic_flink_conf.sh
133142

134143
uses: ./.github/workflows/e2e.yaml
135144
with:
@@ -156,6 +165,7 @@ jobs:
156165
- test_application_kubernetes_ha.sh
157166
- test_application_operations.sh
158167
- test_dynamic_config.sh
168+
- test_dynamic_flink_conf.sh
159169
- test_sessionjob_kubernetes_ha.sh
160170
- test_sessionjob_operations.sh
161171
- test_autoscaler.sh
@@ -169,6 +179,8 @@ jobs:
169179
test: test_autoscaler.sh
170180
- flink-version: v1_16
171181
test: test_dynamic_config.sh
182+
- flink-version: v1_16
183+
test: test_dynamic_flink_conf.sh
172184
- flink-version: v1_16
173185
test: test_flink_operator_ha.sh
174186
- flink-version: v1_16
@@ -177,6 +189,8 @@ jobs:
177189
test: test_batch_job.sh
178190
- flink-version: v1_17
179191
test: test_dynamic_config.sh
192+
- flink-version: v1_17
193+
test: test_dynamic_flink_conf.sh
180194
- flink-version: v1_17
181195
test: test_flink_operator_ha.sh
182196
- flink-version: v1_17
@@ -185,6 +199,8 @@ jobs:
185199
test: test_batch_job.sh
186200
- flink-version: v1_18
187201
test: test_dynamic_config.sh
202+
- flink-version: v1_18
203+
test: test_dynamic_flink_conf.sh
188204
- flink-version: v1_18
189205
test: test_flink_operator_ha.sh
190206
- flink-version: v1_18

‎.github/workflows/e2e.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,9 @@ jobs:
7575
sed -i "s/# kubernetes.operator.leader-election.lease-name: flink-operator-lease/kubernetes.operator.leader-election.lease-name: flink-operator-lease/" helm/flink-kubernetes-operator/conf/flink-conf.yaml
7676
sed -i "s/replicas: 1/replicas: 2/" helm/flink-kubernetes-operator/values.yaml
7777
fi
78+
if [[ "${{ inputs.test }}" == "test_dynamic_config.sh" ]]; then
79+
sed -i "s/flink-conf.yaml: |+/config.yaml: |+/" helm/flink-kubernetes-operator/values.yaml
80+
fi
7881
helm --debug install flink-kubernetes-operator -n ${{ inputs.namespace }} helm/flink-kubernetes-operator --set image.repository=flink-kubernetes-operator --set image.tag=ci-latest ${{ steps.namespace-creator.outputs.EXTRA_HELM_INSTALL_ARGS }}
7982
kubectl wait --for=condition=Available --timeout=120s -n ${{ inputs.namespace }} deploy/flink-kubernetes-operator
8083
kubectl get pods -n ${{ inputs.namespace }}

‎docs/content/docs/operations/helm.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,8 @@ The configurable parameters of the Helm chart and which default values as detail
112112
| defaultConfiguration.create | Whether to enable default configuration to create for flink-kubernetes-operator. | true |
113113
| defaultConfiguration.append | Whether to append configuration files with configs. | true |
114114
| defaultConfiguration.flink-conf.yaml | The default configuration of flink-conf.yaml. | kubernetes.operator.metrics.reporter.slf4j.factory.class: org.apache.flink.metrics.slf4j.Slf4jReporterFactory<br/>kubernetes.operator.metrics.reporter.slf4j.interval: 5 MINUTE<br/>kubernetes.operator.reconcile.interval: 15 s<br/>kubernetes.operator.observer.progress-check.interval: 5 s |
115+
| defaultConfiguration.config.yaml | The newer configuration file format for flink that will enforced in Flink 2.0. Note this was introudced in flink 1.19. | kubernetes.operator.metrics.reporter.slf4j.factory.class: org.apache.flink.metrics.slf4j.Slf4jReporterFactory<br/>kubernetes.operator.metrics.reporter.slf4j.interval: 5 MINUTE<br/>kubernetes.operator.reconcile.interval: 15 s<br/>kubernetes.operator.observer.progress-check.interval: 5 s |
116+
115117
| defaultConfiguration.log4j-operator.properties | The default configuration of log4j-operator.properties. | |
116118
| defaultConfiguration.log4j-console.properties | The default configuration of log4j-console.properties. | |
117119
| metrics.port | The metrics port on the container for default configuration. | |

‎e2e-tests/test_dynamic_config.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ echo "Current operator pod is ${operator_pod}"
3434
create_namespace dynamic
3535

3636
kubectl config set-context --current --namespace="${operator_namespace}"
37-
patch_flink_config '{"data": {"flink-conf.yaml": "kubernetes.operator.watched.namespaces: default,flink,dynamic"}}'
37+
patch_flink_config '{"data": {"config.yaml": "kubernetes.operator.watched.namespaces: default,flink,dynamic"}}'
3838
wait_for_operator_logs "${operator_pod}" "Setting default configuration to {kubernetes.operator.watched.namespaces=default,flink,dynamic}" ${TIMEOUT} || exit 1
3939

4040
echo "Successfully run the dynamic property test"

‎e2e-tests/test_dynamic_flink_conf.sh

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/usr/bin/env bash
2+
################################################################################
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
################################################################################
19+
20+
# This script tests the operator dynamic config on watched namespace:
21+
# 1. Create a new namespace
22+
# 2. Change the watched namespaces by patching on the flink-config-override
23+
# 3. Monitor the operator log to find the watched namespace changed info
24+
SCRIPT_DIR=$(dirname "$(readlink -f "$0")")
25+
source "${SCRIPT_DIR}/utils.sh"
26+
27+
on_exit operator_cleanup_and_exit
28+
29+
TIMEOUT=360
30+
31+
operator_namespace=$(get_operator_pod_namespace)
32+
operator_pod=$(get_operator_pod_name)
33+
echo "Current operator pod is ${operator_pod}"
34+
create_namespace dynamic
35+
36+
kubectl config set-context --current --namespace="${operator_namespace}"
37+
patch_flink_config '{"data": {"flink-conf.yaml": "kubernetes.operator.watched.namespaces: default,flink,dynamic"}}'
38+
wait_for_operator_logs "${operator_pod}" "Setting default configuration to {kubernetes.operator.watched.namespaces=default,flink,dynamic}" ${TIMEOUT} || exit 1
39+
40+
echo "Successfully run the dynamic property test"

‎helm/flink-kubernetes-operator/templates/flink-operator.yaml

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -235,8 +235,13 @@ spec:
235235
configMap:
236236
name: flink-operator-config
237237
items:
238+
{{- if hasKey .Values.defaultConfiguration "flink-conf.yaml" }}
238239
- key: flink-conf.yaml
239240
path: flink-conf.yaml
241+
{{- else }}
242+
- key: config.yaml
243+
path: config.yaml
244+
{{- end }}
240245
- key: log4j-operator.properties
241246
path: log4j-operator.properties
242247
- key: log4j-console.properties
@@ -271,11 +276,25 @@ metadata:
271276
labels:
272277
{{- include "flink-operator.labels" . | nindent 4 }}
273278
data:
279+
config.yaml: |+
280+
{{- if .Values.defaultConfiguration.append }}
281+
{{- $.Files.Get "conf/flink-conf.yaml" | nindent 4 -}}
282+
{{- end }}
283+
{{- if hasKey (.Values.defaultConfiguration) "config.yaml" }}
284+
{{- index (.Values.defaultConfiguration) "config.yaml" | nindent 4 -}}
285+
{{- end }}
286+
{{- if .Values.watchNamespaces }}
287+
kubernetes.operator.watched.namespaces: {{ join "," .Values.watchNamespaces }}
288+
{{- end }}
289+
{{- if index .Values "operatorHealth" }}
290+
kubernetes.operator.health.probe.enabled: true
291+
kubernetes.operator.health.probe.port: {{ .Values.operatorHealth.port }}
292+
{{- end }}
274293
flink-conf.yaml: |+
275294
{{- if .Values.defaultConfiguration.append }}
276295
{{- $.Files.Get "conf/flink-conf.yaml" | nindent 4 -}}
277296
{{- end }}
278-
{{- if index (.Values.defaultConfiguration) "flink-conf.yaml" }}
297+
{{- if hasKey (.Values.defaultConfiguration) "flink-conf.yaml" }}
279298
{{- index (.Values.defaultConfiguration) "flink-conf.yaml" | nindent 4 -}}
280299
{{- end }}
281300
{{- if .Values.watchNamespaces }}

0 commit comments

Comments
 (0)