Merge pull request #337 from dciabrin/OSPRH-17604

Rework retry/timeout defaults to ensure fast service failover

Author: openshift-merge-bot[bot]

templates/galera/bin/mysql_wsrep_notify.sh

@@ -11,9 +11,14 @@ NAMESPACE=$(cat ${SERVICEACCOUNT}/namespace)
 TOKEN=$(cat ${SERVICEACCOUNT}/token)
 CACERT=${SERVICEACCOUNT}/ca.crt
 
-# Retry config
-RETRIES=6
-WAIT=1
+# OSPRH-17604: use default timeout and retry parameters for fast failover
+# default parameters for curl calls to the API server
+: ${WSREP_NOTIFY_CURL_CONNECT_TIMEOUT:=5}
+: ${WSREP_NOTIFY_CURL_MAX_TIME:=30}
+CURL="curl --connect-timeout ${WSREP_NOTIFY_CURL_CONNECT_TIMEOUT} --max-time ${WSREP_NOTIFY_CURL_MAX_TIME}"
+# defaults parameters for retry on error
+: ${WSREP_NOTIFY_RETRIES:=30}
+: ${WSREP_NOTIFY_RETRY_WAIT:=1}
 
 
 ##
@@ -66,7 +71,7 @@ function api_server {
         request="$request -d @-"
     fi
     local output
-    output=$(curl -s --cacert ${CACERT} --header "Content-Type:application/json" --header "Authorization: Bearer ${TOKEN}" --request $request ${APISERVER}/api/v1/namespaces/${NAMESPACE}/services/${service})
+    output=$(${CURL} -s --cacert ${CACERT} --header "Content-Type:application/json" --header "Authorization: Bearer ${TOKEN}" --request $request ${APISERVER}/api/v1/namespaces/${NAMESPACE}/services/${service})
 
     local rc=$?
     if [ $rc != 0 ]; then
@@ -109,8 +114,8 @@ function parse_output {
 # Generic retry logic for an action function
 function retry {
     local action=$1
-    local retries=$RETRIES
-    local wait=$WAIT
+    local retries=$WSREP_NOTIFY_RETRIES
+    local wait=$WSREP_NOTIFY_RETRY_WAIT
     local rc=1
 
     $action
@@ -132,7 +137,7 @@ function retry {
         mysql_probe_state reprobe
     done
     if [ $rc -ne 0 ]; then
-        log_error "Could not run action after ${RETRIES} tries. Stop retrying."
+        log_error "Could not run action after ${WSREP_NOTIFY_RETRIES} tries. Stop retrying."
     fi
     return $rc
 }

tests/chainsaw/tests/service/chainsaw-test.yaml

@@ -74,6 +74,11 @@ spec:
         check:
           # we dont want "ERROR 1047 (08S01) at line 495: WSREP has not yet prepared node for application use"
           (find_first($stdout,'(08S01)') == NULL): true
+    catch: &catch_logs
+    - script:
+        content: |
+          # get full logs for all pods except copy logs from kolla start
+          oc logs -n $NAMESPACE --prefix=true --tail=-1 -l galera/name=openstack | grep -v -e ' INFO:'
 
   - name: Service failover on pod crash
     description: Check that service is failing over when the current endpoint pod crashes
@@ -97,6 +102,7 @@ spec:
         check:
           ($stdout != $endpoint): true
     - script: *no_wsrep_in_failover_check
+    catch: *catch_logs
 
   - name: No failover on random pod restart
     description: Check that service is not impacted when a pod that is not the current endpoint is stopped
@@ -114,3 +120,4 @@ spec:
         check:
           ($stdout == $endpoint): true
     - script: *no_wsrep_in_failover_check
+    catch: *catch_logs

tests/kuttl/common/assert_sample_deployment.yaml

@@ -130,29 +130,3 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: openstack-config-data
----
-apiVersion: kuttl.dev/v1beta1
-kind: TestAssert
-commands:
-  - script: |
-      # when using image digests the containerImage URLs are SHA's so we verify them with a script
-      tupleTemplate='{{ range (index .spec.template.spec.containers 1).env }}{{ .name }}{{ "#" }}{{ .value}}{{"\n"}}{{ end }}'
-      imageTuples=$(oc get -n openstack-operators deployment mariadb-operator-controller-manager -o go-template="$tupleTemplate")
-      # format of imageTuple is: RELATED_IMAGE_MARIADB_<service>#<image URL with SHA> separated by newlines
-      for ITEM in $(echo $imageTuples); do
-        # it is an image
-        if echo $ITEM | grep 'RELATED_IMAGE' &> /dev/null; then
-          NAME=$(echo $ITEM | sed -e 's|^RELATED_IMAGE_MARIADB_\([^_]*\)_.*|\1|')
-          IMG_FROM_ENV=$(echo $ITEM | sed -e 's|^.*#\(.*\)|\1|')
-          template='{{ (index .spec.template.spec.containers 0).image }}'
-          case $NAME in
-            IMAGE)
-              STATEFUL_SET_IMG=$(oc get -n $NAMESPACE statefulset openstack-galera -o go-template="$template")
-              ;;
-          esac
-          if [ "$STATEFUL_SET_IMG" != "$IMG_FROM_ENV" ]; then
-            echo "$NAME image does not equal $VALUE"
-            exit 1
-          fi
-        fi
-      done

tests/kuttl/tests/account_create/05-assert.yaml

@@ -4,7 +4,7 @@ kind: TestAssert
 commands:
   - script: |
       set -e
-      ${MARIADB_KUTTL_DIR:-tests/kuttl/tests}/../common/scripts/check_db_account.sh openstack-galera-0 kuttldb_accounttest someuser dbsecret1
+      ../../common/scripts/check_db_account.sh openstack-galera-0 kuttldb_accounttest someuser dbsecret1
       # ensure db users are configured without TLS connection restriction
       oc rsh -n ${NAMESPACE} -c galera openstack-galera-0 /bin/sh -c 'mysql -uroot -p${DB_ROOT_PASSWORD} -e "show grants for \`someuser\`@\`%\`;"' | grep 'GRANT USAGE' | grep -v 'REQUIRE SSL'
 ---

tests/kuttl/tests/account_create/07-assert.yaml

@@ -3,4 +3,4 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 commands:
   - script: |
-      ${MARIADB_KUTTL_DIR:-tests/kuttl/tests}/../common/scripts/check_db_account.sh openstack-galera-0 kuttldb_accounttest someuser dbsecret1 --reverse
+      ../../common/scripts/check_db_account.sh openstack-galera-0 kuttldb_accounttest someuser dbsecret1 --reverse

tests/kuttl/tests/database_create/03-assert.yaml

@@ -13,8 +13,8 @@ commands:
   # for legacy secret non-present, test that a mariadb username was *not* made
   - script: |
       set -euxo pipefail
-      ${MARIADB_KUTTL_DIR:-tests/kuttl/tests}/../common/scripts/check_db_account.sh openstack-galera-0 kuttldb_utf8 kuttldb_utf_8 12345678 --reverse
+      ../../common/scripts/check_db_account.sh openstack-galera-0 kuttldb_utf8 kuttldb_utf_8 12345678 --reverse
   # for legacy secret present, test that a mariadb username was made
   - script: |
       set -euxo pipefail
-      ${MARIADB_KUTTL_DIR:-tests/kuttl/tests}/../common/scripts/check_db_account.sh openstack-galera-0 kuttldb_legacy_secret kuttldb_legacy_secret dbsecret1
+      ../../common/scripts/check_db_account.sh openstack-galera-0 kuttldb_legacy_secret kuttldb_legacy_secret dbsecret1

tests/kuttl/tests/galera_cluster_restart/04-teardown.yaml

@@ -0,0 +1,13 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+delete:
+  - apiVersion: mariadb.openstack.org/v1beta1
+    kind: Galera
+    name: openstack
+  - apiVersion: v1
+    kind: Secret
+    name: kuttl-galera-tls
+commands:
+  - script: |
+      oc delete -n $NAMESPACE pvc mysql-db-openstack-galera-0 mysql-db-openstack-galera-1 mysql-db-openstack-galera-2
+      for i in `oc get pv | awk '/mysql-db.*galera/ {print $1}'`; do oc patch pv $i -p '{"spec":{"claimRef": null}}'; done

tests/kuttl/tests/galera_deploy_tls/03-teardown.yaml

@@ -7,9 +7,6 @@ delete:
   - apiVersion: v1
     kind: Secret
     name: kuttl-galera-tls
-  - apiVersion: v1
-    kind: Secret
-    name: kuttldb-secret
 commands:
   - script: |
       oc delete -n $NAMESPACE pvc mysql-db-openstack-galera-0 mysql-db-openstack-galera-1 mysql-db-openstack-galera-2

tests/kuttl/tests/galera_log_to_disk/03-teardown.yaml

@@ -0,0 +1,10 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+delete:
+  - apiVersion: mariadb.openstack.org/v1beta1
+    kind: Galera
+    name: openstack
+commands:
+  - script: |
+      oc delete -n $NAMESPACE pvc mysql-db-openstack-galera-0 mysql-db-openstack-galera-1 mysql-db-openstack-galera-2
+      for i in `oc get pv | awk '/mysql-db.*galera/ {print $1}'`; do oc patch pv $i -p '{"spec":{"claimRef": null}}'; done