Skip to content

Commit c790b66

Browse files
raukadahraukadah
committed
[DNM] replace redis with valkey
Note: It is added for testing purpose Signed-off-by: Chandan Kumar (raukadah) <[email protected]>
1 parent 7b84879 commit c790b66

File tree

12 files changed

+342
-11
lines changed

12 files changed

+342
-11
lines changed

‎pkg/redis/volumes.go‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,22 +17,22 @@ func getVolumes(r *redisv1.Redis) []corev1.Volume {
1717
configDataFiles := []corev1.KeyToPath{
1818
{
1919
Key: "sentinel.conf.in",
20-
Path: "var/lib/redis/sentinel.conf.in",
20+
Path: "var/lib/valkey/sentinel.conf.in",
2121
},
2222
{
2323
Key: "redis.conf.in",
24-
Path: "var/lib/redis/redis.conf.in",
24+
Path: "var/lib/valkey/valkey.conf.in",
2525
},
2626
}
2727
if r.Spec.TLS.Enabled() {
2828
configDataFiles = append(configDataFiles, []corev1.KeyToPath{
2929
{
30-
Key: "redis-tls.conf.in",
31-
Path: "var/lib/redis/redis-tls.conf.in",
30+
Key: "valkey-tls.conf.in",
31+
Path: "var/lib/valkey/valkey-tls.conf.in",
3232
},
3333
{
3434
Key: "sentinel-tls.conf.in",
35-
Path: "var/lib/redis/sentinel-tls.conf.in",
35+
Path: "var/lib/valkey/sentinel-tls.conf.in",
3636
},
3737
}...)
3838
}
@@ -96,20 +96,20 @@ func getVolumes(r *redisv1.Redis) []corev1.Volume {
9696
},
9797
Items: []corev1.KeyToPath{
9898
{
99-
Key: "start_redis_replication.sh",
100-
Path: "start_redis_replication.sh",
99+
Key: "start_valkey_replication.sh",
100+
Path: "start_valkey_replication.sh",
101101
},
102102
{
103103
Key: "start_sentinel.sh",
104104
Path: "start_sentinel.sh",
105105
},
106106
{
107-
Key: "redis_probe.sh",
108-
Path: "redis_probe.sh",
107+
Key: "valkey_probe.sh",
108+
Path: "valkey_probe.sh",
109109
},
110110
{
111-
Key: "check_redis_endpoints.sh",
112-
Path: "check_redis_endpoints.sh",
111+
Key: "check_valkey_endpoints.sh",
112+
Path: "check_valkey_endpoints.sh",
113113
},
114114
{
115115
Key: "common.sh",

‎templates/valkey/bin/check_valkey_endpoints.sh‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/bin/bash
2+
3+
. /var/lib/operator-scripts/common.sh
4+
5+
# When the master changed because of a failover, redis notifies this
6+
# script with the following arguments:
7+
# <master-name> <current-role> <state> <old-master> <from-port> <new-master> <to-port>
8+
9+
log "$0 called with arguments: $*"
10+
11+
CLUSTER_NAME=$1
12+
POD_ROLE=$2
13+
STATE=$3
14+
OLD_MASTER=$4
15+
NEW_MASTER=$6
16+
17+
OLD_POD=$(echo $OLD_MASTER | cut -d. -f1)
18+
NEW_POD=$(echo $NEW_MASTER | cut -d. -f1)
19+
20+
if [ "$POD_ROLE" = "leader" ]; then
21+
log "Preparing the endpoint for the failover ${OLD_POD} -> ${NEW_POD}"
22+
23+
log "Removing ${OLD_POD} from the Redis service's endpoint"
24+
remove_pod_label $OLD_POD redis~1master
25+
if [ $? != 0 ]; then
26+
log_error "Could not remove service endpoint. Aborting"
27+
exit 1
28+
fi
29+
30+
log "Setting ${NEW_POD} as the new endpoint for the Redis service"
31+
set_pod_label $NEW_POD redis~1master
32+
if [ $? != 0 ]; then
33+
log_error "Could not add service endpoint. Aborting"
34+
exit 1
35+
fi
36+
else
37+
log "No action taken since we were an observer during the failover"
38+
fi

‎templates/valkey/bin/common.sh‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
# Environment variable common to all scripts
2+
APISERVER=https://kubernetes.default.svc
3+
SERVICEACCOUNT=/var/run/secrets/kubernetes.io/serviceaccount
4+
NAMESPACE=$(cat ${SERVICEACCOUNT}/namespace)
5+
TOKEN=$(cat ${SERVICEACCOUNT}/token)
6+
CACERT=${SERVICEACCOUNT}/ca.crt
7+
8+
TIMEOUT=3
9+
10+
POD_NAME=$HOSTNAME
11+
POD_FQDN=$HOSTNAME.$SVC_FQDN
12+
13+
if test -d /var/lib/config-data/tls; then
14+
REDIS_CLI_CMD="valkey-cli --tls"
15+
REDIS_CONFIG=/var/lib/valkey/valkey-tls.conf
16+
SENTINEL_CONFIG=/var/lib/valkey/sentinel-tls.conf
17+
else
18+
REDIS_CLI_CMD=valkey-cli
19+
REDIS_CONFIG=/var/lib/valkey/valkey.conf
20+
SENTINEL_CONFIG=/var/lib/valkey/sentinel.conf
21+
fi
22+
23+
function log() {
24+
echo "$(date +%F_%H_%M_%S) $*"
25+
}
26+
27+
function log_error() {
28+
echo "$(date +%F_%H_%M_%S) ERROR: $*"
29+
}
30+
31+
function generate_configs() {
32+
# Copying config files except template files
33+
tar -C /var/lib/config-data --exclude '..*' --exclude '*.in' -h -c default | tar -C /var/lib/config-data/generated -x --strip=1
34+
# Generating config files from templates
35+
cd /var/lib/config-data/default
36+
for cfg in $(find -L * -name '*.conf.in'); do
37+
log "Generating config file from template $PWD/${cfg}"
38+
sed -e "s/{ POD_FQDN }/${POD_FQDN}/" "${cfg}" > "/var/lib/config-data/generated/${cfg%.in}"
39+
done
40+
}
41+
42+
function is_bootstrap_pod() {
43+
echo "$1" | grep -qe '-0$'
44+
}
45+
46+
function extract() {
47+
local var="$1"
48+
local output="$2"
49+
# parse curl vars as well as kube api error fields
50+
echo "$output" | awk -F'[:,]' "/\"?${var}\"?:/ {print \$2; exit}"
51+
}
52+
53+
function configure_pod_label() {
54+
local pod="$1"
55+
local patch="$2"
56+
local success="$3"
57+
local curlvars="\nexitcode:%{exitcode}\nerrormsg:%{errormsg}\nhttpcode:%{response_code}\n"
58+
59+
response=$(curl -s -w "${curlvars}" --cacert ${CACERT} --header "Content-Type:application/json-patch+json" --header "Authorization: Bearer ${TOKEN}" --request PATCH --data "$patch" ${APISERVER}/api/v1/namespaces/${NAMESPACE}/pods/${pod})
60+
61+
exitcode=$(extract exitcode "$response")
62+
if [ $exitcode -ne 0 ]; then
63+
errormsg=$(extract errormsg "$response")
64+
log_error "Error when running curl: ${errormsg} (${exitcode})"
65+
return 1
66+
fi
67+
68+
httpcode=$(extract httpcode "$response")
69+
if echo "${httpcode}" | grep -v -E "^${success}$"; then
70+
message=$(extract message "$response")
71+
log_error "Error when calling API server: ${message} (${httpcode})"
72+
return 1
73+
fi
74+
}
75+
76+
function remove_pod_label() {
77+
local pod="$1"
78+
local label="$2"
79+
local patch="[{\"op\": \"remove\", \"path\": \"/metadata/labels/${label}\"}]"
80+
# 200: OK, 422: not found
81+
configure_pod_label $pod "$patch" "(200|422)"
82+
}
83+
84+
function set_pod_label() {
85+
local pod="$1"
86+
local label="$2"
87+
local patch="[{\"op\": \"add\", \"path\": \"/metadata/labels/${label}\", \"value\": \"true\"}]"
88+
# 200: OK
89+
configure_pod_label $pod "$patch" "200"
90+
}

‎templates/valkey/bin/start_sentinel.sh‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/bin/bash
2+
3+
. /var/lib/operator-scripts/common.sh
4+
5+
generate_configs
6+
sudo -E kolla_set_configs
7+
8+
# 1. check if a redis cluster is already running by contacting sentinel
9+
output=$(timeout ${TIMEOUT} $REDIS_CLI_CMD -h ${SVC_FQDN} -p 26379 sentinel master redis)
10+
if [ $? -eq 0 ]; then
11+
master=$(echo "$output" | awk '/^ip$/ {getline; print $0; exit}')
12+
# TODO skip if no master was found
13+
log "Connecting to the existing sentinel cluster (master: $master)"
14+
echo "sentinel monitor redis ${master} 6379 ${SENTINEL_QUORUM}" >> $SENTINEL_CONFIG
15+
exec valkey-sentinel $SENTINEL_CONFIG
16+
fi
17+
18+
# 2. else let the pod's redis server bootstrap a new cluster and monitor it
19+
# (assume we should be the first redis pod)
20+
if is_bootstrap_pod $POD_NAME; then
21+
log "Bootstrapping a new sentinel cluster"
22+
echo "sentinel monitor redis ${POD_FQDN} 6379 ${SENTINEL_QUORUM}" >> $SENTINEL_CONFIG
23+
exec valkey-sentinel $SENTINEL_CONFIG
24+
fi
25+
26+
# 3. else this is an error, exit and let the pod restart and try again
27+
echo "Could not connect to a sentinel cluster"
28+
exit 1

‎templates/valkey/bin/start_valkey_replication.sh‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/bin/bash
2+
3+
. /var/lib/operator-scripts/common.sh
4+
5+
generate_configs
6+
sudo -E kolla_set_configs
7+
8+
# 1. check if a redis cluster is already running by contacting sentinel
9+
output=$(timeout ${TIMEOUT} $REDIS_CLI_CMD -h ${SVC_FQDN} -p 26379 sentinel master redis)
10+
if [ $? -eq 0 ]; then
11+
master=$(echo "$output" | awk '/^ip$/ {getline; print $0; exit}')
12+
# TODO skip if no master was found
13+
log "Connecting to the existing Redis cluster (master: ${master})"
14+
exec valkey-server $REDIS_CONFIG --protected-mode no --replicaof "$master" 6379
15+
fi
16+
17+
# 2. else bootstrap a new cluster (assume we should be the first redis pod)
18+
if is_bootstrap_pod $POD_NAME; then
19+
log "Bootstrapping a new Redis cluster from ${POD_NAME}"
20+
set_pod_label $POD_NAME redis~1master
21+
exec valkey-server $REDIS_CONFIG --protected-mode no
22+
fi
23+
24+
# 3. else this is an error, exit and let the pod restart and try again
25+
echo "Could not connect to a redis cluster"
26+
exit 1

‎templates/valkey/bin/valkey_probe.sh‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash
2+
set -eux
3+
4+
. /var/lib/operator-scripts/common.sh
5+
6+
case "$1" in
7+
readiness)
8+
# ready if we're the master or if we're a slave connected to the current master
9+
output=$($REDIS_CLI_CMD info replication | tr -d '\r')
10+
declare -A state
11+
while IFS=: read -r key value; do state[$key]=$value; done < <(echo "$output")
12+
[[ "${state[role]}" == "master" ]] || [[ "${state[role]}" == "slave" && "${state[master_link_status]}" == "up" ]]
13+
;;
14+
liveness)
15+
$REDIS_CLI_CMD -e ping >/dev/null;;
16+
*)
17+
echo "Invalid probe option '$1'"
18+
exit 1;;
19+
esac

‎templates/valkey/config/config-sentinel.json‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
{
2+
"command": "valkey-sentinel /var/lib/valkey/sentinel.conf",
3+
"config_files": [
4+
{
5+
"dest": "/",
6+
"merge": true,
7+
"preserve_properties": true,
8+
"optional": true,
9+
"source": "/var/lib/config-data/generated/*"
10+
},
11+
{
12+
"source": "/var/lib/config-data/tls/private/valkey.key",
13+
"dest": "/etc/pki/tls/private/valkey.key",
14+
"owner": "valkey",
15+
"perm": "0600",
16+
"optional": true
17+
},
18+
{
19+
"source": "/var/lib/config-data/tls/certs/valkey.crt",
20+
"dest": "/etc/pki/tls/certs/valkey.crt",
21+
"owner": "valkey",
22+
"perm": "0755",
23+
"optional": true
24+
}
25+
],
26+
"permissions": [
27+
{
28+
"owner": "valkey:valkey",
29+
"perm": "0755",
30+
"path": "/var/lib/valkey",
31+
"recursive": true
32+
}
33+
]
34+
}

‎templates/valkey/config/config.json‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
{
2+
"command": "valkey-server /var/lib/valkey/valkey.conf",
3+
"config_files": [
4+
{
5+
"dest": "/",
6+
"merge": true,
7+
"preserve_properties": true,
8+
"optional": true,
9+
"source": "/var/lib/config-data/generated/*"
10+
},
11+
{
12+
"source": "/var/lib/config-data/tls/private/valkey.key",
13+
"dest": "/etc/pki/tls/private/valkey.key",
14+
"owner": "valkey",
15+
"perm": "0600",
16+
"optional": true
17+
},
18+
{
19+
"source": "/var/lib/config-data/tls/certs/valkey.crt",
20+
"dest": "/etc/pki/tls/certs/valkey.crt",
21+
"owner": "valkey",
22+
"perm": "0755",
23+
"optional": true
24+
}
25+
],
26+
"permissions": [
27+
{
28+
"owner": "valkey:valkey",
29+
"path": "/var/lib/valkey",
30+
"recursive": true
31+
}
32+
]
33+
}

‎templates/valkey/config/sentinel-tls.conf.in‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
include /var/lib/valkey/sentinel.conf
2+
3+
port 0
4+
tls-port 26379
5+
tls-cert-file /etc/pki/tls/certs/valkey.crt
6+
tls-key-file /etc/pki/tls/private/valkey.key
7+
tls-ca-cert-file /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
8+
tls-replication yes
9+
tls-auth-clients optional

‎templates/valkey/config/sentinel.conf.in‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
dir /var/lib/valkey
2+
bind { POD_FQDN } localhost
3+
sentinel resolve-hostnames yes
4+
sentinel announce-hostnames yes
5+
sentinel announce-ip { POD_FQDN }
6+
sentinel down-after-milliseconds valkey 10000
7+
sentinel failover-timeout valkey 20000
8+
sentinel client-reconfig-script valkey /var/lib/operator-scripts/check_valkey_endpoints.sh

0 commit comments

Comments
 (0)