Skip to content

Commit 8da9733

Browse files
rahuliitrrahuliitr
authored andcommitted
Helm charts for supporting clustering in ODL
Signed-off-by: Rahul Sharma <Rahul.Sharma@fujitsu.com> Change-Id: Ie128bef4349ece5109e21cd5869b6dd03a245f37
1 parent 08937ee commit 8da9733

File tree

7 files changed

+296
-45
lines changed

7 files changed

+296
-45
lines changed

helm/README.md

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -192,4 +192,35 @@ base on the notes output from heml install, set up port forwarding to first inst
192192
kubectl --namespace default port-forward $POD_NAME 8080:$CONTAINER_PORT
193193
```
194194

195-
from browser, go to `http://127.0.0.1:8080/apidoc/explorer/index.html` then login with `admin/admin`
195+
from browser, go to `http://127.0.0.1:8080/apidoc/explorer/index.html` then login with `admin/admin`
196+
197+
## Clustering
198+
199+
Opendaylight can be started as an Akka based cluster.
200+
To start Opendaylight as a cluster, following parametes in values.yaml need to be updated:
201+
```
202+
1. replicaCount : The value should be at least 3 or higher number (need to be an odd number).
203+
2. config.isClusterDeployment: The value should be set to true
204+
3. autoscaling.enabled: This should be disabled (i.e.; enabled: false)
205+
```
206+
Once set, you can run helm install as above:
207+
208+
```
209+
helm install sdnc opendaylight
210+
211+
Output:
212+
/home/rahul/packaging/helm$ kubectl get pods
213+
NAME READY STATUS RESTARTS AGE
214+
sdnc-opendaylight-1 1/1 Running 0 25m
215+
sdnc-opendaylight-2 1/1 Running 0 25m
216+
sdnc-opendaylight-0 1/1 Running 0 25m
217+
```
218+
219+
NOTE: If the variable `config.isClusterDeployment` is set to `false` with replicaCount set to 3, you will have 3 independent ODL instances deployed (but not clustered).
220+
221+
### Cluster specific APIs
222+
Once the cluster is up, you can use jolokia APIs to query the status of cluster:
223+
- `http://{{vm-ip}}:{{odl-restconf-port}}/jolokia/read/org.opendaylight.controller:Category=Shards,name=member-<leader-id>-shard-default-operational,type=DistributedOperationalDatastore`
224+
-- NOTE: To know the leader-id in the above query, use: `http://{{vm-ip}}:{{odl-restconf-port}}/jolokia/read/org.opendaylight.controller:type=DistributedOperationalDatastore,Category=ShardManager,name=shard-manager-operational`
225+
226+
For more details, refer here: https://docs.opendaylight.org/en/stable-phosphorus/getting-started-guide/clustering.html#cluster-monitoring
Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
# SPDX-License-Identifier: EPL-1.0
2+
##############################################################################
3+
# Copyright (c) 2021 The Linux Foundation and others.
4+
#
5+
# All rights reserved. This program and the accompanying materials
6+
# are made available under the terms of the Eclipse Public License v1.0
7+
# which accompanies this distribution, and is available at
8+
# http://www.eclipse.org/legal/epl-v10.html
9+
##############################################################################
10+
#!/bin/bash -e
11+
12+
function setup_cluster(){
13+
if [ -z $ODL_REPLICAS ]; then
14+
echo "ODL_REPLICAS is not configured in Env field"
15+
exit
16+
fi
17+
18+
hm=$(hostname)
19+
echo "Enable cluster for host: ${hm}"
20+
21+
## For hostname viz; odl-opendaylight-1,
22+
## node_name will be 'odl-opendaylight' and node_index '1'
23+
node_name=${hm%-*};
24+
node_index=${hm##*-};
25+
node_list="${node_name}-0.{{ include "opendaylight.fullname" . }}.{{ .Release.Namespace }}";
26+
27+
for ((i=1;i<${ODL_REPLICAS};i++));
28+
do
29+
node_list="${node_list} ${node_name}-$i.{{ include "opendaylight.fullname" . }}.{{ .Release.Namespace }}"
30+
done
31+
32+
${BASEDIR}/bin/configure_cluster.sh $((node_index+1)) ${node_list}
33+
}
34+
35+
set -x
36+
37+
mountpath="{{ .Values.persistence.mountPath }}"
38+
BASEDIR="{{ .Values.config.odl_basedir }}"
39+
odl_prefix="/opt/opendaylight"
40+
41+
if [[ ! -d "$mountpath/snapshots" ]];then
42+
mkdir -p $mountpath/snapshots
43+
fi
44+
45+
if [[ ! -d "$mountpath/data" ]];then
46+
mkdir -p $mountpath/data
47+
fi
48+
49+
if [[ ! -d "$mountpath/segmented-journal" ]];then
50+
mkdir -p $mountpath/segmented-journal
51+
fi
52+
53+
if [[ ! -d "$mountpath/daexim" ]];then
54+
mkdir -p $mountpath/daexim
55+
fi
56+
57+
if [[ ! -L "$odl_prefix/snapshots" ]];then
58+
rm -rf $odl_prefix/snapshots && ln -s $mountpath/snapshots $odl_prefix/snapshots
59+
fi
60+
61+
if [[ ! -L "$odl_prefix/data" ]];then
62+
rm -rf $odl_prefix/data && ln -s $mountpath/data $odl_prefix/data
63+
fi
64+
65+
if [[ ! -L "$odl_prefix/segmented-journal" ]];then
66+
rm -rf $odl_prefix/segmented-journal && ln -s $mountpath/segmented-journal $odl_prefix/segmented-journal
67+
fi
68+
69+
if [[ ! -L "$odl_prefix/daexim" ]];then
70+
rm -rf $odl_prefix/daexim && ln -s $mountpath/daexim $odl_prefix/daexim
71+
fi
72+
73+
sed -i "s/\(featuresBoot= \|featuresBoot = \)/featuresBoot = ${FEATURES},/g" ${BASEDIR}/etc/org.apache.karaf.features.cfg
74+
cat ${BASEDIR}/etc/org.apache.karaf.features.cfg
75+
76+
ODL_REPLICAS=${ODL_REPLICAS:-1}
77+
IS_CLUSTER_ENABLED=${IS_CLUSTER_ENABLED:-false}
78+
SLEEP_TIME=${SLEEP_TIME:-30}
79+
80+
if $IS_CLUSTER_ENABLED; then
81+
${BASEDIR}/bin/start;
82+
echo "Waiting ${SLEEP_TIME} seconds for OpenDaylight to initialize";
83+
sleep ${SLEEP_TIME};
84+
setup_cluster;
85+
echo "Restart ODL after cluster configuration";
86+
${BASEDIR}/bin/stop;
87+
sleep 20;
88+
fi
89+
90+
echo "Starting OpenDaylight"
91+
${BASEDIR}/bin/karaf run
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
# SPDX-License-Identifier: EPL-1.0
2+
##############################################################################
3+
# Copyright (c) 2021 The Linux Foundation and others.
4+
#
5+
# All rights reserved. This program and the accompanying materials
6+
# are made available under the terms of the Eclipse Public License v1.0
7+
# which accompanies this distribution, and is available at
8+
# http://www.eclipse.org/legal/epl-v10.html
9+
##############################################################################
10+
odl-cluster-data {
11+
akka {
12+
remote {
13+
artery {
14+
enabled = on
15+
canonical.hostname = "127.0.0.1"
16+
canonical.port = 2550
17+
}
18+
19+
use-passive-connections = off
20+
# when under load we might trip a false positive on the failure detector
21+
# transport-failure-detector {
22+
# heartbeat-interval = 4 s
23+
# acceptable-heartbeat-pause = 16s
24+
# }
25+
}
26+
27+
actor {
28+
debug {
29+
autoreceive = on
30+
lifecycle = on
31+
unhandled = on
32+
fsm = on
33+
event-stream = on
34+
}
35+
}
36+
37+
cluster {
38+
# Using artery.
39+
seed-nodes = ["akka://opendaylight-cluster-data@127.0.0.1:2550"]
40+
41+
seed-node-timeout = {{ .Values.cluster.akka.seedNodeTimeout }}
42+
43+
downing-provider-class = "akka.cluster.sbr.SplitBrainResolverProvider"
44+
45+
roles = ["member-1"]
46+
47+
}
48+
49+
persistence {
50+
# By default the snapshots/journal directories live in KARAF_HOME. You can choose to put it somewhere else by
51+
# modifying the following two properties. The directory location specified may be a relative or absolute path.
52+
# The relative path is always relative to KARAF_HOME.
53+
54+
# snapshot-store.local.dir = "target/snapshots"
55+
# journal.leveldb.dir = "target/journal"
56+
57+
journal {
58+
leveldb {
59+
# Set native = off to use a Java-only implementation of leveldb.
60+
# Note that the Java-only version is not currently considered by Akka to be production quality.
61+
62+
# native = off
63+
}
64+
65+
journal-plugin-fallback {
66+
circuit-breaker {
67+
max-failures = {{ .Values.cluster.akka.circuitBreaker.maxFailures }}
68+
call-timeout = {{ .Values.cluster.akka.circuitBreaker.callTimeout }}
69+
reset-timeout = {{ .Values.cluster.akka.circuitBreaker.resetTimeout }}
70+
}
71+
recovery-event-timeout = {{ .Values.cluster.akka.recoveryEventTimeout }}
72+
}
73+
74+
snapshot-store-plugin-fallback {
75+
circuit-breaker {
76+
max-failures = {{ .Values.cluster.akka.circuitBreaker.maxFailures }}
77+
call-timeout = {{ .Values.cluster.akka.circuitBreaker.callTimeout }}
78+
reset-timeout = {{ .Values.cluster.akka.circuitBreaker.resetTimeout }}
79+
}
80+
recovery-event-timeout = {{ .Values.cluster.akka.recoveryEventTimeout }}
81+
}
82+
}
83+
84+
# Use lz4 compression for LocalSnapshotStore snapshots
85+
snapshot-store.local.use-lz4-compression = false
86+
# Size of blocks for lz4 compression: 64KB, 256KB, 1MB or 4MB
87+
snapshot-store.local.lz4-blocksize = 256KB
88+
}
89+
disable-default-actor-system-quarantined-event-handling = "false"
90+
}
91+
}
92+

helm/opendaylight/templates/configmap.yaml

Lines changed: 11 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -13,46 +13,15 @@
1313
apiVersion: v1
1414
kind: ConfigMap
1515
metadata:
16-
name: {{ include "opendaylight.fullname" . }}
16+
name: {{ include "opendaylight.fullname" . }}-scripts
1717
data:
18-
startodl.sh: |
19-
#!/bin/bash
20-
mountpath="{{ .Values.persistence.mountPath }}"
21-
BASEDIR="{{ .Values.config.odl_basedir }}"
22-
odl_prefix="/opt/opendaylight"
23-
24-
if [[ ! -d "$mountpath/snapshots" ]];then
25-
mkdir -p $mountpath/snapshots
26-
fi
27-
28-
if [[ ! -d "$mountpath/data" ]];then
29-
mkdir -p $mountpath/data
30-
fi
31-
32-
if [[ ! -d "$mountpath/segmented-journal" ]];then
33-
mkdir -p $mountpath/segmented-journal
34-
fi
35-
36-
if [[ ! -d "$mountpath/daexim" ]];then
37-
mkdir -p $mountpath/daexim
38-
fi
39-
40-
if [[ ! -L "$odl_prefix/snapshots" ]];then
41-
rm -rf $odl_prefix/snapshots && ln -s $mountpath/snapshots $odl_prefix/snapshots
42-
fi
43-
44-
if [[ ! -L "$odl_prefix/data" ]];then
45-
rm -rf $odl_prefix/data && ln -s $mountpath/data $odl_prefix/data
46-
fi
47-
48-
if [[ ! -L "$odl_prefix/segmented-journal" ]];then
49-
rm -rf $odl_prefix/segmented-journal && ln -s $mountpath/segmented-journal $odl_prefix/segmented-journal
50-
fi
51-
52-
if [[ ! -L "$odl_prefix/daexim" ]];then
53-
rm -rf $odl_prefix/daexim && ln -s $mountpath/daexim $odl_prefix/daexim
54-
fi
55-
56-
sed -i "s/\(featuresBoot= \|featuresBoot = \)/featuresBoot = ${FEATURES},/g" ${BASEDIR}/etc/org.apache.karaf.features.cfg
57-
cat ${BASEDIR}/etc/org.apache.karaf.features.cfg
58-
${BASEDIR}/bin/karaf run
18+
{{ tpl (.Files.Glob "resources/bin/*").AsConfig . | indent 2 }}
19+
---
20+
{{- if .Values.config.isClusterDeployment }}
21+
apiVersion: v1
22+
kind: ConfigMap
23+
metadata:
24+
name: {{ include "opendaylight.fullname" . }}-conf
25+
data:
26+
{{ tpl (.Files.Glob "resources/conf/*").AsConfig . | indent 2 }}
27+
{{- end }}

helm/opendaylight/templates/service.yaml

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,30 @@ metadata:
1616
name: {{ include "opendaylight.fullname" . }}
1717
labels:
1818
{{- include "opendaylight.labels" . | nindent 4 }}
19+
annotations:
20+
service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
1921
spec:
2022
type: {{ .Values.service.type }}
2123
ports:
2224
- port: {{ .Values.service.port }}
2325
targetPort: http
2426
protocol: TCP
2527
name: http
28+
{{- if eq .Values.service.type "NodePort" }}
29+
nodePort: {{ .Values.service.nodePortHttp }}
30+
{{- end }}
31+
{{- if .Values.config.isClusterDeployment }}
32+
- port: {{ .Values.service.clusterPort }}
33+
targetPort: cluster
34+
protocol: TCP
35+
name: cluster
36+
{{- if eq .Values.service.type "NodePort" }}
37+
nodePort: {{ .Values.service.nodePortCluster }}
38+
{{- end }}
39+
{{- end }}
40+
{{- if eq .Values.service.type "ClusterIP" }}
2641
clusterIP: None
42+
{{- end }}
2743
selector:
2844
{{- include "opendaylight.selectorLabels" . | nindent 4 }}
45+
publishNotReadyAddresses: true

helm/opendaylight/templates/statefulset.yaml

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ spec:
1818
{{- if not .Values.autoscaling.enabled }}
1919
replicas: {{ .Values.replicaCount }}
2020
{{- end }}
21+
podManagementPolicy: Parallel
2122
serviceName: {{ include "opendaylight.fullname" . }}
2223
selector:
2324
matchLabels:
@@ -45,6 +46,18 @@ spec:
4546
volumeMounts:
4647
- name: {{ .Values.persistence.volName }}
4748
mountPath: {{ .Values.persistence.mountPath }}
49+
{{- if .Values.config.isClusterDeployment }}
50+
- name: inject-cluster-related-conf
51+
image: busybox
52+
command: ["/bin/sh"]
53+
args: ["-c", "cp /config-input/akka.conf /config/akka.conf"]
54+
volumeMounts:
55+
- mountPath: /config-input/akka.conf
56+
name: config-input
57+
subPath: akka.conf
58+
- mountPath: /config
59+
name: update-conf
60+
{{- end }}
4861
containers:
4962
- name: {{ .Chart.Name }}
5063
securityContext:
@@ -61,10 +74,19 @@ spec:
6174
value: "-Xms{{.Values.config.javaOptions.minMemory}} -Xmx{{.Values.config.javaOptions.maxMemory}}"
6275
- name: EXTRA_JAVA_OPTS
6376
value: "-XX:+UseG1GC -XX:MaxGCPauseMillis={{.Values.config.javaOptions.maxGCPauseMillis}} -XX:ParallelGCThreads={{.Values.config.javaOptions.parallelGCThreads}} -XX:+ParallelRefProcEnabled -XX:+UseStringDeduplication {{.Values.config.javaOptions.gcLogOptions}}"
77+
- name: ODL_REPLICAS
78+
value: "{{ .Values.replicaCount }}"
79+
- name: IS_CLUSTER_ENABLED
80+
value: "{{ .Values.config.isClusterDeployment }}"
6481
ports:
6582
- name: http
6683
containerPort: {{ .Values.service.port }}
6784
protocol: TCP
85+
{{- if .Values.config.isClusterDeployment }}
86+
- name: cluster
87+
containerPort: {{ .Values.service.clusterPort }}
88+
protocol: TCP
89+
{{- end }}
6890
readinessProbe:
6991
tcpSocket:
7092
port: {{ .Values.service.port }}
@@ -77,6 +99,11 @@ spec:
7799
mountPath: {{ .Values.persistence.mountPath }}
78100
- name: scripts
79101
mountPath: /scripts
102+
{{- if .Values.config.isClusterDeployment }}
103+
- mountPath: {{ .Values.cluster.salConfigDir }}/{{ .Values.cluster.salConfigVersion}}/sal-clustering-config-{{ .Values.cluster.salConfigVersion}}-akkaconf.xml
104+
name: update-conf
105+
subPath: akka.conf
106+
{{- end }}
80107
{{- with .Values.nodeSelector }}
81108
nodeSelector:
82109
{{- toYaml . | nindent 8 }}
@@ -92,7 +119,16 @@ spec:
92119
volumes:
93120
- name: scripts
94121
configMap:
95-
name: {{ include "opendaylight.fullname" . }}
122+
name: {{ include "opendaylight.fullname" . }}-scripts
123+
{{- if .Values.config.isClusterDeployment }}
124+
- name: config-input
125+
configMap:
126+
name: {{ include "opendaylight.fullname" . }}-conf
127+
defaultMode: 0755
128+
- name: update-conf
129+
emptyDir:
130+
medium: Memory
131+
{{- end }}
96132
{{ if not .Values.persistence.enabled }}
97133
- name: {{ .Values.persistence.volName }}
98134
emptyDir: {}

0 commit comments

Comments
 (0)