Skip to content

Commit cfe6567

Browse files
pengzhoumlPeng Zhou
pengzhouml
and
Peng Zhou
authored
MLE-14633 bug fix for failed over security forest not working (#262)
* MLE-14633: POC for failover bug fix * fix hardcoded image string in TestMlAdminSecrets * add test based routing * change function name for init_marklogic * MLE-15529 Fix bug in TestSeparateEDnode and TestTlsOnEDnode tests * update retry times for setting group * remove debug log messages * remove empty line --------- Co-authored-by: Peng Zhou <[email protected]>
1 parent b687ebb commit cfe6567

File tree

3 files changed

+107
-56
lines changed

3 files changed

+107
-56
lines changed

‎charts/templates/configmap-scripts.yaml

Lines changed: 104 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -169,10 +169,11 @@ data:
169169
#! /bin/bash
170170
# Refer to https://docs.marklogic.com/guide/admin-api/cluster#id_10889 for cluster joining process
171171
172-
mkdir -p /tmp/marklogic
173172
N_RETRY=10
174173
RETRY_INTERVAL=5
175-
HOST_FQDN="$(hostname).${MARKLOGIC_FQDN_SUFFIX}"
174+
HOSTNAME=$(cat /etc/hostname)
175+
HOST_FQDN="${HOSTNAME}.${MARKLOGIC_FQDN_SUFFIX}"
176+
ML_KUBERNETES_FILE_PATH="/var/opt/MarkLogic/Kubernetes"
176177
177178
# HTTP_PROTOCOL could be http or https
178179
HTTP_PROTOCOL="http"
@@ -183,7 +184,7 @@ data:
183184
fi
184185
185186
IS_BOOTSTRAP_HOST=false
186-
if [[ "$(hostname)" == *-0 ]]; then
187+
if [[ "${HOSTNAME}" == *-0 ]]; then
187188
echo "IS_BOOTSTRAP_HOST true"
188189
IS_BOOTSTRAP_HOST=true
189190
else
@@ -344,16 +345,16 @@ data:
344345
# return values: 0 - successfully initialized
345346
# 1 - host not reachable
346347
################################################################
347-
function wait_until_marklogic_ready {
348+
function init_marklogic {
348349
local host=$1
349350
info "wait until $host is ready"
350-
timestamp=$( curl -s --anyauth \
351+
timestamp=$( curl -s --anyauth -m 4 \
351352
--user "${MARKLOGIC_ADMIN_USERNAME}":"${MARKLOGIC_ADMIN_PASSWORD}" \
352-
http://${host}:8001/admin/v1/timestamp )
353+
http://localhost:8001/admin/v1/timestamp )
353354
if [ -z "${timestamp}" ]; then
354355
info "${host} - not responding yet"
355-
sleep 5s
356-
wait_until_marklogic_ready $host
356+
sleep 10s
357+
init_marklogic $host
357358
return 0
358359
else
359360
info "${host} - responding with $timestamp"
@@ -365,7 +366,7 @@ data:
365366
-i -X POST -H "Content-type:application/json" \
366367
-d "${LICENSE_PAYLOAD}" \
367368
--user "${MARKLOGIC_ADMIN_USERNAME}":"${MARKLOGIC_ADMIN_PASSWORD}" \
368-
http://${host}:8001/admin/v1/init \
369+
http://localhost:8001/admin/v1/init \
369370
)
370371
if [ "${response_code}" = "202" ]; then
371372
info "${host} - init called, restart triggered"
@@ -437,7 +438,7 @@ data:
437438
info "${MARKLOGIC_BOOTSTRAP_HOST} - bootstrap security already initialized"
438439
return 0
439440
else
440-
info "${MARKLOGIC_BOOTSTRAP_HOST} - initializing bootstrap security"
441+
info "initializing bootstrap security"
441442
442443
# Get last restart timestamp directly before instance-admin call to verify restart after
443444
timestamp=$( \
@@ -454,7 +455,7 @@ data:
454455
455456
restart_check "${MARKLOGIC_BOOTSTRAP_HOST}" "${timestamp}"
456457
457-
info "${MARKLOGIC_BOOTSTRAP_HOST} - bootstrap security initialized"
458+
info "bootstrap security initialized"
458459
return 0
459460
fi
460461
}
@@ -502,7 +503,7 @@ data:
502503
503504
# process to join the host
504505
# Wait until the group is ready
505-
retry_count=5
506+
retry_count=10
506507
while [ $retry_count -gt 0 ]; do
507508
GROUP_RESP_CODE=$( curl --anyauth -m 20 -s -o /dev/null -w "%{http_code}" $HTTPS_OPTION -X GET $HTTP_PROTOCOL://${MARKLOGIC_BOOTSTRAP_HOST}:8002/manage/v2/groups/${MARKLOGIC_GROUP} --anyauth --user ${MARKLOGIC_ADMIN_USERNAME}:${MARKLOGIC_ADMIN_PASSWORD} )
508509
info "GROUP_RESP_CODE: $GROUP_RESP_CODE"
@@ -521,32 +522,32 @@ data:
521522
fi
522523
done
523524
524-
info "${hostname} - joining cluster of group ${MARKLOGIC_GROUP}"
525+
info "joining cluster of group ${MARKLOGIC_GROUP}"
525526
MARKLOGIC_GROUP_PAYLOAD="group=${MARKLOGIC_GROUP}"
526-
curl_retry_validate false "http://${hostname}:8001/admin/v1/server-config" 200 \
527+
curl_retry_validate false "http://localhost:8001/admin/v1/server-config" 200 \
527528
"-o" "/tmp/host.xml" "-X" "GET" "-H" "Accept: application/xml"
528529
529-
info "${hostname} - getting cluster-config from bootstrap host"
530+
info "getting cluster-config from bootstrap host"
530531
curl_retry_validate false "$HTTP_PROTOCOL://${MARKLOGIC_BOOTSTRAP_HOST}:8001/admin/v1/cluster-config" 200 \
531532
"--anyauth" "--user" "${MARKLOGIC_ADMIN_USERNAME}:${MARKLOGIC_ADMIN_PASSWORD}" \
532533
"-X" "POST" "-d" "${MARKLOGIC_GROUP_PAYLOAD}" \
533534
"--data-urlencode" "server-config@/tmp/host.xml" \
534535
"-H" "Content-type: application/x-www-form-urlencoded" \
535536
"-o" "/tmp/cluster.zip" $HTTPS_OPTION
536537
537-
timestamp=$(curl -s "http://${hostname}:8001/admin/v1/timestamp" )
538+
timestamp=$(curl -s "http://localhost:8001/admin/v1/timestamp" )
538539
539-
info "${hostname} - joining cluster of group ${MARKLOGIC_GROUP}"
540-
curl_retry_validate false "http://${hostname}:8001/admin/v1/cluster-config" 202 \
540+
info "joining cluster of group ${MARKLOGIC_GROUP}"
541+
curl_retry_validate false "http://localhost:8001/admin/v1/cluster-config" 202 \
541542
"-o" "/dev/null" \
542543
"-X" "POST" "-H" "Content-type: application/zip" \
543544
"--data-binary" "@/tmp/cluster.zip"
544545
545546
# 202 causes restart
546-
info "${hostname} - restart triggered"
547-
restart_check "${hostname}" "${timestamp}"
547+
info "restart triggered"
548+
restart_check "localhost" "${timestamp}"
548549
549-
info "${hostname} - joined group ${MARKLOGIC_GROUP}"
550+
info "joined group ${MARKLOGIC_GROUP}"
550551
}
551552
552553
################################################################
@@ -558,8 +559,8 @@ data:
558559
local LOCAL_HTTP_PROTOCOL LOCAL_HTTPS_OPTION
559560
LOCAL_HTTP_PROTOCOL="http"
560561
LOCAL_HTTPS_OPTION=""
561-
protocol=$(get_current_host_protocol $MARKLOGIC_BOOTSTRAP_HOST)
562-
if [[ $protocol == "https" ]]; then
562+
bootstrap_protocol=$(get_current_host_protocol $MARKLOGIC_BOOTSTRAP_HOST)
563+
if [[ $bootstrap_protocol == "https" ]]; then
563564
LOCAL_HTTP_PROTOCOL="https"
564565
LOCAL_HTTPS_OPTION="-k"
565566
fi
@@ -571,12 +572,12 @@ data:
571572
# check if host is already in and get the current cluster
572573
curl_retry_validate false "$LOCAL_HTTP_PROTOCOL://${MARKLOGIC_BOOTSTRAP_HOST}:8002/manage/v2/hosts/${HOST_FQDN}/properties?format=xml" 200 \
573574
"--anyauth" "--user" "${MARKLOGIC_ADMIN_USERNAME}:${MARKLOGIC_ADMIN_PASSWORD}" \
574-
"-o" "/tmp/marklogic/groups.out" $LOCAL_HTTPS_OPTION
575+
"-o" "/tmp/groups.out" $LOCAL_HTTPS_OPTION
575576
576577
response_code=$?
577578
if [ "${response_code}" = "200" ]; then
578579
current_group=$( \
579-
cat "/tmp/marklogic/groups.out" |
580+
cat "/tmp/groups.out" |
580581
grep "group" |
581582
sed 's%^.*<group.*>\(.*\)</group>.*$%\1%' \
582583
)
@@ -786,47 +787,98 @@ data:
786787
787788
log "Info: removing cert keys"
788789
rm -f /run/secrets/marklogic-certs/*.key
789-
}
790+
}
790791
791792
792-
info "Start configuring MarkLogic for $HOST_FQDN"
793-
info "Bootstrap host: $MARKLOGIC_BOOTSTRAP_HOST"
793+
function configure_path_based_routing {
794+
# Authentication configuration when path based is used
795+
if [[ $PATH_BASED_ROUTING == "true" ]]; then
796+
log "Info: path based routing is set. Adapting authentication method"
797+
resp=$(curl --anyauth -w "%{http_code}" --user $MARKLOGIC_ADMIN_USERNAME:$MARKLOGIC_ADMIN_PASSWORD -m 20 -s -X PUT -H "Content-type: application/json" -d '{"authentication":"basic"}' http://localhost:8002/manage/v2/servers/Admin/properties?group-id=${MARKLOGIC_GROUP})
798+
log "Info: Admin-Servers response code: $resp"
799+
resp=$(curl --anyauth -w "%{http_code}" --user $MARKLOGIC_ADMIN_USERNAME:$MARKLOGIC_ADMIN_PASSWORD -m 20 -s -X PUT -H "Content-type: application/json" -d '{"authentication":"basic"}' http://localhost:8002/manage/v2/servers/App-Services/properties?group-id=${MARKLOGIC_GROUP})
800+
log "Info: App Service response code: $resp"
801+
resp=$(curl --anyauth -w "%{http_code}" --user $MARKLOGIC_ADMIN_USERNAME:$MARKLOGIC_ADMIN_PASSWORD -m 20 -s -X PUT -H "Content-type: application/json" -d '{"authentication":"basic"}' http://localhost:8002/manage/v2/servers/Manage/properties?group-id=${MARKLOGIC_GROUP})
802+
log "Info: Manage response code: $resp"
803+
log "Info: Default App-Servers authentication set to basic auth"
804+
else
805+
log "Info: This is not the boostrap host or path based routing is not set. Skipping authentication configuration"
806+
fi
807+
#End of authentication configuration
808+
}
809+
810+
function set_status_file {
811+
mkdir -p $ML_KUBERNETES_FILE_PATH
812+
fqdn=$(hostname -f)
813+
status_file="$ML_KUBERNETES_FILE_PATH/status.txt"
814+
group_name="${MARKLOGIC_GROUP}"
815+
group_xdqp_ssl_enabled="${XDQP_SSL_ENABLED}"
816+
https_enabled="${MARKLOGIC_JOIN_TLS_ENABLED}"
817+
echo "fqdn=${fqdn}" > $status_file
818+
echo "group_name=${group_name}" >> $status_file
819+
echo "group_xdqp_ssl_enabled=${group_xdqp_ssl_enabled}" >> $status_file
820+
echo "https_enabled=${https_enabled}" >> $status_file
821+
}
822+
823+
function check_status_file_for_nonbootstrap {
824+
if [[ -f "$ML_KUBERNETES_FILE_PATH/status.txt" ]]; then
825+
log "Info: status file exists. Skip configuration"
826+
exit 0
827+
else
828+
log "Info: status file does not exist. Continue"
829+
fi
830+
}
831+
832+
function check_status_file_for_boostrap {
833+
if [[ -f "$ML_KUBERNETES_FILE_PATH/status.txt" ]]; then
834+
new_group_name="${MARKLOGIC_GROUP}"
835+
new_group_xdqp_ssl_enabled="${XDQP_SSL_ENABLED}"
836+
new_https_enabled="${MARKLOGIC_JOIN_TLS_ENABLED}"
837+
source "$ML_KUBERNETES_FILE_PATH/status.txt"
838+
if [[ "$new_group_name" == "$group_name" ]] && [[ "$new_group_xdqp_ssl_enabled" == "$group_xdqp_ssl_enabled" ]] && [[ "$new_https_enabled" == "$https_enabled" ]]; then
839+
log "No change in values file. Skip configuration"
840+
exit 0
841+
else
842+
log "Info: changes made in values file. Continue Configuration"
843+
fi
844+
else
845+
return 0
846+
fi
847+
}
794848
795849
# Wait for current pod ready
796-
wait_until_marklogic_ready $HOST_FQDN
850+
851+
info "Start configuring MarkLogic for $HOST_FQDN"
852+
info "Bootstrap host: $MARKLOGIC_BOOTSTRAP_HOST"
797853
798854
# Only do this if the bootstrap host is in the statefulset we are configuring
799-
if [[ "${MARKLOGIC_CLUSTER_TYPE}" = "bootstrap" && "${HOST_FQDN}" = "${MARKLOGIC_BOOTSTRAP_HOST}" ]]; then
800-
sleep 2s
801-
init_security_db
802-
configure_group
803-
else
855+
if [[ "$IS_BOOTSTRAP_HOST" == "true" ]]; then
856+
check_status_file_for_boostrap
857+
init_marklogic $HOST_FQDN
858+
if [[ "${MARKLOGIC_CLUSTER_TYPE}" == "bootstrap" ]]; then
859+
log "Info: bootstrap host is ready"
860+
init_security_db
861+
configure_group
862+
else
863+
log "Info: bootstrap host is ready"
864+
configure_group
865+
join_cluster $HOST_FQDN
866+
fi
867+
configure_path_based_routing
868+
else
869+
check_status_file_for_nonbootstrap
870+
init_marklogic $HOST_FQDN
804871
wait_bootstrap_ready
805-
configure_group
806872
join_cluster $HOST_FQDN
807873
fi
808874
809-
sleep 5s
810-
811-
# Authentication configuration when path based is used
812-
if [[ "$IS_BOOTSTRAP_HOST" == "true" ]] && [[ $PATH_BASED_ROUTING == "true" ]]; then
813-
log "Info: path based routing is set. Adapting authentication method"
814-
resp=$(curl --anyauth -w "%{http_code}" --user $MARKLOGIC_ADMIN_USERNAME:$MARKLOGIC_ADMIN_PASSWORD -m 20 -s -X PUT -H "Content-type: application/json" -d '{"authentication":"basic"}' http://localhost:8002/manage/v2/servers/Admin/properties?group-id=${MARKLOGIC_GROUP})
815-
log "Info: Admin-Servers response code: $resp"
816-
resp=$(curl --anyauth -w "%{http_code}" --user $MARKLOGIC_ADMIN_USERNAME:$MARKLOGIC_ADMIN_PASSWORD -m 20 -s -X PUT -H "Content-type: application/json" -d '{"authentication":"basic"}' http://localhost:8002/manage/v2/servers/App-Services/properties?group-id=${MARKLOGIC_GROUP})
817-
log "Info: App Service response code: $resp"
818-
resp=$(curl --anyauth -w "%{http_code}" --user $MARKLOGIC_ADMIN_USERNAME:$MARKLOGIC_ADMIN_PASSWORD -m 20 -s -X PUT -H "Content-type: application/json" -d '{"authentication":"basic"}' http://localhost:8002/manage/v2/servers/Manage/properties?group-id=${MARKLOGIC_GROUP})
819-
log "Info: Manage response code: $resp"
820-
log "Info: Default App-Servers authentication set to basic auth"
821-
else
822-
log "Info: This is not the boostrap host or path based routing is not set. Skipping authentication configuration"
823-
fi
824-
#End of authentication configuration
825-
826875
if [[ $MARKLOGIC_JOIN_TLS_ENABLED == "true" ]]; then
876+
log "configuring tls"
827877
configure_tls
828878
fi
829879
880+
set_status_file
881+
830882
info "helm script completed"
831883
832884
root-rootless-upgrade.sh: |

‎test/e2e/admin_secrets_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,8 @@ func TestMlAdminSecrets(t *testing.T) {
4444
SetValues: map[string]string{
4545
"persistence.enabled": "true",
4646
"replicaCount": "1",
47-
"image.repository": "ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi-rootless",
48-
"image.tag": "latest-11",
47+
"image.repository": imageRepo,
48+
"image.tag": imageTag,
4949
"auth.adminUsername": "admin",
5050
"auth.adminPassword": "admin",
5151
"auth.walletPassword": "admin",

‎test/e2e/separate_nodes_test.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ func VerifyDnodeConfig(t *testing.T, dnodePodName string, kubectlOptions *k8s.Ku
6868
t.Errorf("Bootstrap does not exists on cluster")
6969
}
7070

71-
dnodeEndpoint := fmt.Sprintf("%s://%s/manage/v2/hosts?format=json", protocol, tunnel.Endpoint())
71+
dnodeEndpoint := fmt.Sprintf("%s://%s/manage/v2/groups/dnode/properties?format=json", protocol, tunnel.Endpoint())
7272
t.Log("====Verifying xdqp-ssl-enabled is set to true for dnode group")
7373
resp, err = client.R().
7474
Get(dnodeEndpoint)
@@ -239,7 +239,6 @@ func TestSeparateEDnode(t *testing.T) {
239239

240240
// wait until the pod is in ready status
241241
k8s.WaitUntilPodAvailable(t, kubectlOptions, dnodePodName, 15, 20*time.Second)
242-
243242
bootstrapHost, err := VerifyDnodeConfig(t, dnodePodName, kubectlOptions, "http")
244243
if err != nil {
245244
t.Errorf(err.Error())

0 commit comments

Comments
 (0)