functions

#!/bin/bash

GIT_COMMIT=$(git rev-parse HEAD)
GIT_BRANCH=${VERSION:-$(git rev-parse --abbrev-ref HEAD | sed -e 's^/^-^g; s^[.]^-^g;' | sed -e 's/_/-/g' | tr '[:upper:]' '[:lower:]')}
API="psmdb.percona.com/v1"
OPERATOR_VERSION="$(grep 'crVersion' $(realpath $(dirname ${BASH_SOURCE[0]})/../deploy/cr.yaml) | awk '{print $2}')"
IMAGE=${IMAGE:-"perconalab/percona-server-mongodb-operator:${GIT_BRANCH}"}
IMAGE_MONGOD=${IMAGE_MONGOD:-"perconalab/percona-server-mongodb-operator:main-mongod8.0"}
IMAGE_MONGOD_CHAIN=${IMAGE_MONGOD_CHAIN:-$'
perconalab/percona-server-mongodb-operator:main-mongod6.0
perconalab/percona-server-mongodb-operator:main-mongod7.0
perconalab/percona-server-mongodb-operator:main-mongod8.0'}
IMAGE_BACKUP=${IMAGE_BACKUP:-"perconalab/percona-server-mongodb-operator:main-backup"}
SKIP_BACKUPS_TO_AWS_GCP_AZURE=${SKIP_BACKUPS_TO_AWS_GCP_AZURE:-1}
PMM_SERVER_VER=${PMM_SERVER_VER:-"9.9.9"}
IMAGE_PMM_CLIENT=${IMAGE_PMM_CLIENT:-"percona/pmm-client:2.44.1-1"}
IMAGE_PMM_SERVER=${IMAGE_PMM_SERVER:-"perconalab/pmm-server:dev-latest"}
IMAGE_PMM3_CLIENT=${IMAGE_PMM3_CLIENT:-"perconalab/pmm-client:3-dev-latest"}
IMAGE_PMM3_SERVER=${IMAGE_PMM3_SERVER:-"perconalab/pmm-server:3-dev-latest"}
CERT_MANAGER_VER="1.19.1"
MINIO_VER="5.4.0"
CHAOS_MESH_VER="2.7.1"
UPDATE_COMPARE_FILES=${UPDATE_COMPARE_FILES:-0}
DELETE_CRD_ON_START=${DELETE_CRD_ON_START:-1}
SKIP_DELETE=${SKIP_DELETE:-0}
tmp_dir=$(mktemp -d)
sed=$(which gsed || which sed)
date=$(which gdate || which date)

test_name=$(basename $test_dir)
namespace="${test_name}-${RANDOM}"
replica_namespace="${test_name}-replica-${RANDOM}"
conf_dir=$(realpath $test_dir/../conf || :)
src_dir=$(realpath $test_dir/../..)
logs_dir=$(realpath $test_dir/../logs || :)

if [[ ${ENABLE_LOGGING} == "true" ]]; then
	if [ ! -d "${logs_dir}" ]; then
		mkdir "${logs_dir}"
	fi
	exec &> >(tee ${logs_dir}/${test_name}.log)
	echo "Log: ${logs_dir}/${test_name}.log"
fi

if [ -f "$conf_dir/cloud-secret.yml" ]; then
	SKIP_BACKUPS_TO_AWS_GCP_AZURE=''
fi

if oc get projects 2>/dev/null; then
	OPENSHIFT=4
fi

if kubectl get nodes | grep "^minikube" >/dev/null; then
	export MINIKUBE=1
fi

minikube_sleep() {
	sleep_time=${1:-10}
	if [[ $MINIKUBE == 1 ]]; then
		sleep $sleep_time
	fi
}

if [ $(kubectl version -o json | jq -r '.serverVersion.gitVersion' | grep "-eks-") ]; then
	EKS=1
else
	EKS=0
fi

if [ $(kubectl version -o json | jq -r '.serverVersion.gitVersion' | grep "gke") ]; then
	GKE=1
else
	GKE=0
fi
KUBE_VERSION=$(kubectl version -o json | jq -r '.serverVersion.major + "." + .serverVersion.minor' | $sed -r 's/[^0-9.]+//g')

log() {
	set +o xtrace
	echo "[$(date +%Y-%m-%dT%H:%M:%S%z)]" $*
	set_debug
}

sleep_with_log() {
	local d=$1
	local msg=$2

	log "${msg}. sleeping for ${d} seconds"
	sleep ${d}
}

set_debug() {
	if [[ ${DEBUG_TESTS} == 1 ]]; then
		set -o xtrace
	else
		set +o xtrace
	fi
}

version_gt() {
	# return true if kubernetes version equal or greater than desired
	if [ $(echo "${KUBE_VERSION} >= $1" | bc -l) -eq 1 ]; then
		return 0
	else
		return 1
	fi
}

apply_s3_storage_secrets() {
	desc 'create secrets for cloud storages'
	if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then
		kubectl_bin apply \
			-f $conf_dir/minio-secret.yml \
			-f $conf_dir/cloud-secret.yml
	else
		kubectl_bin apply \
			-f $conf_dir/minio-secret.yml
	fi
}

set_kube_ctx() {
	local namespace=$1

	kubectl_bin config set-context $(kubectl_bin config current-context) --namespace="$namespace"
}

create_namespace() {
	local namespace="$1"
	local skip_clean_namespace="$2"

	if [[ ${CLEAN_NAMESPACE} == 1 ]] && [[ -z ${skip_clean_namespace} ]]; then
		destroy_chaos_mesh
		desc 'cleaned up all old namespaces'
		kubectl_bin get ns \
			| grep -E -v "^kube-|^default|Terminating|psmdb-operator|openshift|^gke-|^gmp-|^NAME" \
			| awk '{print$1}' \
			| xargs kubectl delete ns &
	fi

	if [ -n "${OPENSHIFT}" ]; then
		desc 'cleaned up all old namespaces'
		if [ -n "$OPERATOR_NS" -a $(oc get project "$OPERATOR_NS" -o json | jq -r '.metadata.name') ]; then
			oc delete --grace-period=0 --force=true project "$namespace" && sleep 120 || :
		else
			oc delete project "$namespace" && sleep 40 || :
		fi

		desc "create namespace $namespace"
		oc new-project "$namespace"
		oc project "$namespace"
		oc adm policy add-scc-to-user hostaccess -z default || :
	else

		desc "cleaned up old namespaces $namespace"
		kubectl_bin delete namespace "$namespace" --ignore-not-found || :
		kubectl_bin wait --for=delete namespace "$namespace" || :

		desc "create namespace $namespace"
		kubectl_bin create namespace "$namespace"
		set_kube_ctx ${namespace}
	fi
}

get_operator_pod() {
	kubectl_bin get pods \
		--selector=name=percona-server-mongodb-operator \
		-o 'jsonpath={.items[].metadata.name}' ${OPERATOR_NS:+-n $OPERATOR_NS}
}

wait_operator_pod() {
	local pod=$1

	set +o xtrace
	retry=0
	echo -n "waiting for pod/$pod to be ready"
	until kubectl_bin get pod/$pod ${OPERATOR_NS:+-n $OPERATOR_NS} -o jsonpath='{.status.containerStatuses[0].ready}' 2>/dev/null | grep --quiet 'true'; do
		sleep 1
		echo -n .
		let retry+=1
		if [[ $retry -ge 360 ]]; then
			kubectl_bin describe pod/$pod
			kubectl_bin logs $pod ${OPERATOR_NS:+-n $OPERATOR_NS}
			echo max retry count $retry reached. something went wrong with operator or kubernetes cluster
			exit 1
		fi
	done
	echo ".OK"
	set_debug
}

wait_pod() {
	local pod=$1

	set +o xtrace
	retry=0
	echo -n "waiting for pod/$pod to be ready"
	until kubectl_bin get pod/$pod -o jsonpath='{.status.containerStatuses[0].ready}' 2>/dev/null | grep --quiet 'true'; do
		sleep 1
		echo -n .
		let retry+=1
		if [ $retry -ge 360 ]; then
			kubectl_bin describe pod/$pod
			kubectl_bin logs $pod
			kubectl_bin logs ${OPERATOR_NS:+-n $OPERATOR_NS} $(get_operator_pod) \
				| grep -v 'level=info' \
				| grep -v 'level=debug' \
				| grep -v 'Getting tasks for pod' \
				| grep -v 'Getting pods from source' \
				| tail -100
			echo max retry count $retry reached. something went wrong with operator or kubernetes cluster
			exit 1
		fi
	done
	echo ".OK"
	set_debug
}

wait_cron() {
	local backup=$1

	set +o xtrace
	retry=0
	echo -n $backup
	until kubectl_bin get cronjob/$backup -o jsonpath='{.status.lastScheduleTime}' 2>/dev/null | grep 'T'; do
		sleep 1
		echo -n .
		let retry+=1
		if [ $retry -ge 360 ]; then
			kubectl_bin logs ${OPERATOR_NS:+-n $OPERATOR_NS} $(get_operator_pod) \
				| grep -v 'level=info' \
				| grep -v 'level=debug' \
				| grep -v 'Getting tasks for pod' \
				| grep -v 'Getting pods from source' \
				| tail -100
			echo max retry count $retry reached. something went wrong with operator or kubernetes cluster
			exit 1
		fi
	done
	set_debug
}

wait_backup_agent() {
	local agent_pod=$1

	set +o xtrace
	local retry=0
	echo -n "waiting for pbm-agent to be ready in ${agent_pod}..."
	until kubectl_bin logs $agent_pod -c backup-agent | grep "listening for the commands"; do
		sleep 5
		echo -n .
		let retry+=1
		if [ $retry -ge 360 ]; then
			kubectl_bin logs $agent_pod -c backup-agent | tail -100
			echo max retry count $retry reached. something went wrong with operator or kubernetes cluster
			exit 1
		fi
	done
	echo
	set_debug
}

print_operator_logs() {
	kubectl_bin logs ${OPERATOR_NS:+-n $OPERATOR_NS} $(get_operator_pod)
}

wait_backup() {
	local backup_name=$1
	local target_state=${2:-"ready"}

	set +o xtrace
	retry=0
	echo -n "waiting for ${backup_name} to reach ${target_state} state"
	local current_status=
	until [[ ${current_status} == ${target_state} ]]; do
		sleep 1
		echo -n .
		let retry+=1
		current_status=$(kubectl_bin get psmdb-backup $backup_name -o jsonpath='{.status.state}')
		if [[ $retry -ge 600 || ${current_status} == 'error' ]]; then
			print_operator_logs | tail -n 200
			kubectl_bin get psmdb-backup
			kubectl_bin describe psmdb-backup/${backup_name}
			echo "Backup object psmdb-backup/${backup_name} is in ${current_status} state."
			exit 1
		fi
	done
	echo '.OK'
	set_debug
}

wait_for_pbm_operations() {
	local cluster=$1

	set +o xtrace
	echo -n "waiting for PBM operation to finish"
	retry=0
	until [[ $(kubectl_bin exec ${cluster}-rs0-0 -c backup-agent -- pbm status -o json -s running | jq -r .running.opID) == null ]]; do
		if [ $retry -ge 540 ]; then
			echo max retry count $retry reached. something went wrong with PBM operations
			exit 1
		fi
		echo -n .
		sleep 5
	done
	echo '.OK'
	set_debug
}

run_restore() {
	local backup_name=$1
	local restore_file=${2:-$test_dir/conf/restore.yml}

	log "running restore restore-$backup_name"

	cat ${restore_file} \
		| $sed -e "s/name:/name: restore-$backup_name/" \
		| $sed -e "s/backupName:/backupName: $backup_name/" \
		| kubectl_bin apply -f -
}

run_restore_backupsource() {
	local backupName=$1
	local backupDest=$2
	local storageName=$3

	desc "run restore restore-$backupName from backup $backupName destination is $backupDest"
	if [ -z "$storageName" ]; then
		cat $test_dir/conf/restore-backupsource.yml \
			| $sed -e "s/name:/name: restore-$backupName/" \
			| $sed -e "s|BACKUP-NAME|$backupDest|" \
			| $sed -e "/storageName/d" \
			| kubectl_bin apply -f -

		return
	fi

	cat $test_dir/conf/restore-backupsource.yml \
		| $sed -e "s/name:/name: restore-$backupName/" \
		| $sed -e "s|BACKUP-NAME|$backupDest|" \
		| $sed -e "s/storageName:/storageName: $storageName/" \
		| kubectl_bin apply -f -
}

wait_deployment() {
	local name=$1

	sleep 10
	retry=0
	echo -n $name
	until kubectl_bin get deployment $name ${OPERATOR_NS:+-n $OPERATOR_NS} >/dev/null \
		&& [ "$(kubectl_bin get deployment $name -o jsonpath='{.status.replicas}' ${OPERATOR_NS:+-n $OPERATOR_NS})" == "$(kubectl_bin get deployment $name -o jsonpath='{.status.readyReplicas}' ${OPERATOR_NS:+-n $OPERATOR_NS})" ]; do
		sleep 1
		echo -n .
		let retry+=1
		if [ $retry -ge 360 ]; then
			kubectl_bin logs ${OPERATOR_NS:+-n $OPERATOR_NS} $(get_operator_pod) \
				| grep -v 'level=info' \
				| grep -v 'level=debug' \
				| grep -v 'Getting tasks for pod' \
				| grep -v 'Getting pods from source' \
				| tail -100
			echo max retry count $retry reached. something went wrong with operator or kubernetes cluster
			exit 1
		fi
	done
}

simple_data_check() {
	local cluster_name=${1}
	let last_pod="$2-1" || :
	local isSharded=${3:-0}
	local cluster_pfx=$4

	if [ $isSharded -eq 1 ]; then
		sleep 10 # give time for start mongos
		wait_cluster_consistency "${cluster_name}"
		compare_mongos_cmd "find" "myApp:myPass@$cluster$cluster_pfx.$namespace"
	else
		for i in $(seq 0 $last_pod); do
			compare_mongo_cmd "find" "myApp:myPass@${cluster_name}${cluster_pfx}-${i}.${cluster_name}${cluster_pfx}.$namespace"
		done
	fi
}

get_mongod_pods() {
	local cluster=$1

	kubectl_bin get pod \
		--no-headers \
		-l app.kubernetes.io/instance=${cluster} \
		-l app.kubernetes.io/component=mongod | awk '{print $1}'

}

collect_physical_restore_logs() {
	local cluster=$1
	local restore=$2

	for pod in $(get_mongod_pods ${cluster}); do
		desc "pbm-agent logs from ${pod}"
		kubectl_bin exec ${pod} -- cat /tmp/pbm-agent.log || echo "failed to get /tmp/pbm-agent.log from ${pod}"
	done
}

is_physical_backup() {
	local backup=$1

	if [[ $(kubectl_bin get psmdb-backup ${backup} -o jsonpath={.status.type}) == "physical" ]]; then
		return 0
	fi

	return 1
}

wait_restore() {
	local backup_name=$1
	local cluster_name=$2
	local target_state=${3:-"ready"}
	local wait_cluster_consistency=${4:-1}
	local wait_time=${5:-1780}
	local ok_if_ready=${6:-0}

	set +o xtrace
	# We need to run wait till object is created, otherwise wait fails at once
	echo -n "Waiting for the psmdb-restore/restore-$backup_name object to be created"
	retry_object=0
	until kubectl_bin get psmdb-restore restore-$backup_name >/dev/null 2>&1; do
		echo -n .
		let retry_object+=1
		if [[ ${retry_object} -ge 60 ]]; then
			echo "psmdb-restore/restore-$backup_name object was not created."
			exit 1
		fi
		sleep 1
	done
	echo ".OK"

	echo -n "Waiting psmdb-restore/restore-${backup_name} to reach state \"${target_state}\" "
	retry=0
	retry_count=$((wait_time / 60))
	until kubectl wait psmdb-restore restore-${backup_name} --for=jsonpath='{.status.state}'=${target_state} --timeout=60s >/dev/null 2>&1; do
		echo -n .
		let retry+=1
		current_state=$(kubectl_bin get psmdb-restore restore-$backup_name -o jsonpath='{.status.state}')
		if [[ ${ok_if_ready} == 1 && ${current_state} == 'ready' ]]; then
			echo ".OK after ${retry} minutes"
			break
		fi
		if [[ ${retry} -ge ${retry_count} || ${current_state} == 'error' ]]; then
			if is_physical_backup ${backup_name}; then
				collect_physical_restore_logs
			fi

			kubectl_bin logs ${OPERATOR_NS:+-n $OPERATOR_NS} $(get_operator_pod) \
				| grep "restore-${backup_name}" \
				| tail -100

			kubectl_bin get psmdb-restore restore-${backup_name} -o yaml
			kubectl_bin describe psmdb-restore restore-${backup_name}

			echo "Restore object restore-${backup_name} is in ${current_state} state after ${retry} minutes."
			echo something went wrong with operator or kubernetes cluster
			exit 1
		fi
	done
	echo ".OK after ${retry} minutes"
	set_debug

	if [[ $wait_cluster_consistency -eq 1 ]]; then
		wait_cluster_consistency "${cluster_name}"
	fi
}

apply_rbac() {
	local operator_namespace=${OPERATOR_NS:-'psmdb-operator'}
	local rbac=${1:-'rbac'}

	cat ${src_dir}/deploy/${rbac}.yaml \
		| sed -e "s^namespace: .*^namespace: $operator_namespace^" \
		| kubectl_bin apply ${OPERATOR_NS:+-n $OPERATOR_NS} -f -
}

deploy_operator() {
	desc "start PSMDB operator: ${IMAGE}"

	local cr_file
	if [ -f "${test_dir}/conf/crd.yaml" ]; then
		cr_file="${test_dir}/conf/crd.yaml"
	else
		cr_file="${src_dir}/deploy/crd.yaml"
	fi

	kubectl_bin apply --server-side --force-conflicts -f "${cr_file}"
	if [ -n "$OPERATOR_NS" ]; then
		apply_rbac cw-rbac
		yq eval '
			(.spec.template.spec.containers[].image = "'${IMAGE}'") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' ${src_dir}/deploy/cw-operator.yaml \
			| kubectl_bin apply ${OPERATOR_NS:+-n $OPERATOR_NS} -f -
	else
		apply_rbac rbac
		yq eval '
			(.spec.template.spec.containers[].image = "'${IMAGE}'") |
			((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
			((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' ${src_dir}/deploy/operator.yaml \
			| kubectl_bin apply -f -
	fi
	sleep 20
	wait_operator_pod $(get_operator_pod)
	echo "Print operator info from log"
	kubectl_bin logs ${OPERATOR_NS:+-n $OPERATOR_NS} $(get_operator_pod) | grep 'Manager starting up'
}

deploy_operator_gh() {
	local git_tag="$1"

	desc 'start operator'
	kubectl_bin apply -f "https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml" --server-side
	local rbac_yaml="rbac"
	local operator_yaml="operator"
	if [ -n "${OPERATOR_NS}" ]; then
		rbac_yaml="cw-rbac"
		operator_yaml="cw-operator"
	fi
	kubectl_bin apply -f "https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/${rbac_yaml}.yaml"
	curl -s "https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/${operator_yaml}.yaml" >"${tmp_dir}/${operator_yaml}_${git_tag}.yaml"

	yq eval '
		(.spec.template.spec.containers[].image = "'${IMAGE}'") |
		((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
		((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' "${tmp_dir}/${operator_yaml}_${git_tag}.yaml" \
		| kubectl_bin apply ${OPERATOR_NS:+-n $OPERATOR_NS} -f -

	sleep 2
	wait_operator_pod "$(get_operator_pod)"
	echo "Print operator info from log"
	kubectl_bin logs ${OPERATOR_NS:+-n $OPERATOR_NS} $(get_operator_pod) | grep 'Manager starting up'
}

deploy_operator_gh_release() {
	local git_tag="$1"

	desc "start operator ${git_tag}"
	kubectl_bin apply -f "https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml" --server-side
	local rbac_yaml="rbac"
	local operator_yaml="operator"
	if [ -n "${OPERATOR_NS}" ]; then
		rbac_yaml="cw-rbac"
		operator_yaml="cw-operator"
	fi
	kubectl_bin apply -f "https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/${rbac_yaml}.yaml"
	curl -s "https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/${operator_yaml}.yaml" >"${tmp_dir}/${operator_yaml}_${git_tag}.yaml"

	yq eval '
		((.. | select(.[] == "DISABLE_TELEMETRY")) |= .value="true") |
		((.. | select(.[] == "LOG_LEVEL")) |= .value="DEBUG")' "${tmp_dir}/${operator_yaml}_${git_tag}.yaml" \
		| kubectl_bin apply ${OPERATOR_NS:+-n $OPERATOR_NS} -f -

	sleep 20
	wait_operator_pod "$(get_operator_pod)"
	echo "Print operator info from log"
	kubectl_bin logs ${OPERATOR_NS:+-n $OPERATOR_NS} $(get_operator_pod) | grep 'Manager starting up'
}

deploy_minio() {
	desc 'install Minio'
	helm uninstall minio-service || :
	helm repo remove minio || :
	helm repo add minio https://charts.min.io/
	# kubectl_bin delete pvc minio-service --force
	retry 10 60 helm install minio-service \
		--version $MINIO_VER \
		--set replicas=1 \
		--set mode=standalone \
		--set resources.requests.memory=256Mi \
		--set rootUser=rootuser \
		--set rootPassword=rootpass123 \
		--set "users[0].accessKey=some-access-key" \
		--set "users[0].secretKey=some-secret-key" \
		--set "users[0].policy=consoleAdmin" \
		--set service.type=ClusterIP \
		--set configPathmc=/tmp/.minio/ \
		--set persistence.size=2G \
		--set securityContext.enabled=false \
		minio/minio
	MINIO_POD=$(kubectl_bin get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}')
	wait_pod $MINIO_POD

	if [ -n "$OPERATOR_NS" ]; then
		kubectl_bin create svc -n ${OPERATOR_NS} externalname minio-service --external-name="minio-service.${namespace}.svc.cluster.local" --tcp="9000"
	fi

	create_minio_bucket operator-testing
}

create_minio_bucket() {
	local bucket=$1

	kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- \
		bash -c "AWS_ACCESS_KEY_ID=some-access-key \
		AWS_SECRET_ACCESS_KEY=some-secret-key \
		AWS_DEFAULT_REGION=us-east-1 \
        /usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://${bucket}"
}

deploy_vault() {
	local name=${1:-vault-service}
	if [[ $# -gt 0 ]]; then
		shift
	fi

	desc "install Vault $name"
	helm uninstall "$name" || :
	helm repo remove hashicorp || :
	helm repo add hashicorp https://helm.releases.hashicorp.com

	destroy_vault "$name"

	if [[ -n ${OPENSHIFT} ]]; then
		oc patch clusterrole system:auth-delegator --type='json' -p '[{"op":"add","path":"/rules/-", "value":{"apiGroups":["security.openshift.io"], "resourceNames": ["privileged"], "resources":["securitycontextconstraints"],"verbs":["use"]}}]'

		retry 10 60 helm install "$name" hashicorp/vault \
			--disable-openapi-validation \
			--set dataStorage.enabled=false \
			--set global.openshift=true \
			--set injector.image.repository="docker.io/hashicorp/vault-k8s" \
			--set injector.agentImage.repository="docker.io/hashicorp/vault" \
			--set server.image.repository="docker.io/hashicorp/vault" "$@"
	else
		retry 10 60 helm install "$name" hashicorp/vault \
			--disable-openapi-validation \
			--set dataStorage.enabled=false "$@"
	fi

	until kubectl_bin get pod/"$name"-0 -o jsonpath='{.status.phase}' 2>/dev/null | grep 'Running'; do
		sleep 1
	done

	sleep 5
	kubectl_bin exec pod/"$name"-0 -- vault operator init -key-shares=1 -key-threshold=1 -format=json >"${tmp_dir}"/vault-init
	local unsealKey
	unsealKey=$(jq -r ".unseal_keys_b64[]" <"${tmp_dir}"/vault-init)
	local token
	token=$(jq -r ".root_token" <"${tmp_dir}"/vault-init)
	kubectl_bin exec pod/"$name"-0 -- vault operator unseal "${unsealKey}"
	kubectl_bin exec -it pod/"$name"-0 -- sh <<EOF
vault login "${token}"
vault secrets enable -path secret kv-v2
EOF
	kubectl_bin create secret generic vault-secret --from-literal=token="${token}"
}

destroy_vault() {
	local name=${1:-vault-service}
	local vault_ns
	vault_ns="$(helm list --all-namespaces --filter "$name" | tail -n1 | awk -F' ' '{print $2}' | sed 's/NAMESPACE//')"

	desc 'destroy vault'
	for i in $(kubectl api-resources | grep vault | awk '{print $1}'); do timeout 30 kubectl delete "${i}" --all --all-namespaces || :; done
	if [ -n "${vault_ns}" ]; then
		helm uninstall "$name" --namespace "${vault_ns}" || :
	fi

	# shellcheck disable=SC2046
	timeout 30 kubectl delete clusterrolebinding $(kubectl get clusterrolebinding -l "app.kubernetes.io/instance=$name" -o jsonpath='{range .items[*]}{.metadata.name}{" "}{end}') || :
	# shellcheck disable=SC2046
	timeout 30 kubectl delete clusterrole $(kubectl get clusterrole -l "app.kubernetes.io/instance=$name" -o jsonpath='{range .items[*]}{.metadata.name}{" "}{end}') || :
	# shellcheck disable=SC2046
	timeout 30 kubectl delete mutatingwebhookconfiguration $(kubectl get mutatingwebhookconfiguration -l "app.kubernetes.io/instance=$name" -o jsonpath='{range .items[*]}{.metadata.name}{" "}{end}') || :
}

deploy_chaos_mesh() {
	local chaos_mesh_ns=$1

	destroy_chaos_mesh

	desc 'install chaos-mesh'
	helm repo add chaos-mesh https://charts.chaos-mesh.org
	helm install chaos-mesh chaos-mesh/chaos-mesh --namespace=${chaos_mesh_ns} --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock --set dashboard.create=false --version $CHAOS_MESH_VER
	sleep 10
}

destroy_chaos_mesh() {
	local chaos_mesh_ns=$(helm list --all-namespaces --filter chaos-mesh | tail -n1 | awk -F' ' '{print $2}' | sed 's/NAMESPACE//')

	desc 'destroy chaos-mesh'
	if [ -n "${chaos_mesh_ns}" ]; then
		helm uninstall --wait --timeout 60s chaos-mesh --namespace ${chaos_mesh_ns} || :
	fi
	timeout 30 kubectl delete MutatingWebhookConfiguration $(kubectl get MutatingWebhookConfiguration | grep 'chaos-mesh' | awk '{print $1}') || :
	timeout 30 kubectl delete ValidatingWebhookConfiguration $(kubectl get ValidatingWebhookConfiguration | grep 'chaos-mesh' | awk '{print $1}') || :
	timeout 30 kubectl delete ValidatingWebhookConfiguration $(kubectl get ValidatingWebhookConfiguration | grep 'validate-auth' | awk '{print $1}') || :
	for i in $(kubectl api-resources | grep chaos-mesh | awk '{print $1}'); do
		kubectl get ${i} --all-namespaces --no-headers -o custom-columns=Kind:.kind,Name:.metadata.name,NAMESPACE:.metadata.namespace \
			| while read -r line; do
				local kind=$(echo "$line" | awk '{print $1}')
				local name=$(echo "$line" | awk '{print $2}')
				local namespace=$(echo "$line" | awk '{print $3}')
				kubectl patch $kind $name -n $namespace --type=merge -p '{"metadata":{"finalizers":[]}}' || :
			done
		timeout 30 kubectl delete ${i} --all --all-namespaces || :
	done
	timeout 30 kubectl delete crd $(kubectl get crd | grep 'chaos-mesh.org' | awk '{print $1}') || :
	timeout 30 kubectl delete clusterrolebinding $(kubectl get clusterrolebinding | grep 'chaos-mesh' | awk '{print $1}') || :
	timeout 30 kubectl delete clusterrole $(kubectl get clusterrole | grep 'chaos-mesh' | awk '{print $1}') || :
}

retry() {
	local max=$1
	local delay=$2
	shift 2 # cut delay and max args
	local n=1

	until "$@"; do
		if [[ $n -ge $max ]]; then
			echo "The command '$@' has failed after $n attempts."
			exit 1
		fi
		((n++))
		sleep $delay
	done
}

wait_for_running() {
	local name="$1"
	let last_pod="$(($2 - 1))" || :
	local check_cluster_readyness="${3:-true}"

	set_debug
	local rs_name=${name/*-/}
	local cluster_name=${name/-$rs_name/}
	for i in $(seq 0 $last_pod); do
		if [[ $i -eq $last_pod && $(kubectl_bin get psmdb ${cluster_name} -o jsonpath='{.spec.replsets[?(@.name=="'${rs_name}'")].arbiter.enabled}') == "true" ]]; then
			wait_pod ${name}-arbiter-0
		else
			wait_pod ${name}-${i}
		fi
	done
	if [[ $(kubectl_bin get psmdb ${cluster_name} -o jsonpath='{.spec.replsets[?(@.name=="'${rs_name}'")].nonvoting.enabled}') == "true" ]]; then
		last_pod=$(($(kubectl_bin get psmdb ${cluster_name} -o jsonpath='{.spec.replsets[?(@.name=="'${rs_name}'")].nonvoting.size}') - 1))
		for i in $(seq 0 $last_pod); do
			wait_pod ${name}-nv-${i}
		done
	fi
	if [[ $(kubectl_bin get psmdb ${cluster_name} -o jsonpath='{.spec.replsets[?(@.name=="'${rs_name}'")].hidden.enabled}') == "true" ]]; then
		last_pod=$(($(kubectl_bin get psmdb ${cluster_name} -o jsonpath='{.spec.replsets[?(@.name=="'${rs_name}'")].hidden.size}') - 1))
		for i in $(seq 0 $last_pod); do
			wait_pod ${name}-hidden-${i}
		done
	fi
	sleep 10
	if [[ ${check_cluster_readyness} == "true" ]]; then
		set +x
		echo -n "Waiting for cluster readyness"
		local timeout=0
		until [[ $(kubectl_bin get psmdb ${cluster_name} -o jsonpath={.status.state}) == "ready" ]]; do
			sleep 1
			timeout=$((timeout + 1))
			echo -n '.'
			if [[ ${timeout} -gt 1500 ]]; then
				kubectl_bin logs ${OPERATOR_NS:+-n $OPERATOR_NS} $(get_operator_pod) \
					| grep -v 'level=info' \
					| grep -v 'level=debug' \
					| grep -v 'Getting tasks for pod' \
					| grep -v 'Getting pods from source' \
					| tail -200
				echo
				echo "Waiting timeout has been reached. Exiting..."
				exit 1
			fi
		done
		echo
		set_debug
	fi
}

wait_for_delete() {
	local res="$1"
	local wait_time=${2:-60}

	set +o xtrace
	echo -n "waiting for $res to be deleted"
	retry=0
	until (kubectl_bin get $res || :) 2>&1 | grep NotFound; do
		sleep 1
		echo -n .
		let retry+=1
		if [ $retry -ge $wait_time ]; then
			kubectl_bin logs ${OPERATOR_NS:+-n $OPERATOR_NS} $(get_operator_pod) \
				| grep -v 'level=info' \
				| grep -v 'level=debug' \
				| grep -v 'Getting tasks for pod' \
				| grep -v 'Getting pods from source' \
				| tail -100
			echo max retry count $retry reached. something went wrong with operator or kubernetes cluster
			exit 1
		fi
	done
	set_debug
}

compare_generation() {
	local generation="$1"
	local resource_type="$2"
	local resource_name="$3"
	local current_generation

	current_generation="$(kubectl_bin get ${resource_type} "${resource_name}" -o jsonpath='{.metadata.generation}')"
	if [[ ${generation} != "${current_generation}" ]]; then
		echo "Generation for ${resource_type}/${resource_name} is: ${current_generation}, but should be: ${generation}"
		exit 1
	fi
}

compare_kubectl() {
	local resource="$1"
	local postfix="$2"
	local skip_generation_check="$3"
	local expected_result=${test_dir}/compare/${resource//\//_}${postfix}.yml
	local new_result="${tmp_dir}/${resource//\//_}.yml"

	if [ -n "$OPENSHIFT" -a -f ${expected_result//.yml/-oc.yml} ]; then
		desc "OPENSHIFT"
		expected_result=${expected_result//.yml/-oc.yml}
		if [ "$OPENSHIFT" = 4 -a -f ${expected_result//-oc.yml/-4-oc.yml} ]; then
			expected_result=${expected_result//-oc.yml/-4-oc.yml}
		fi
	fi

	kubectl_bin get -o yaml ${resource} \
		| yq eval '
			del(.metadata.ownerReferences[].apiVersion) |
			del(.metadata.managedFields) |
			del(.. | select(has("creationTimestamp")).creationTimestamp) |
			del(.. | select(has("namespace")).namespace) |
			del(.. | select(has("uid")).uid) |
			del(.metadata.resourceVersion) |
			del(.spec.template.spec.containers[].env[] | select(.name == "NAMESPACE")) |
			del(.metadata.selfLink) |
			del(.metadata.annotations."cloud.google.com/neg") |
			del(.metadata.annotations."kubectl.kubernetes.io/last-applied-configuration") |
			del(.. | select(has("image")).image) |
			del(.. | select(has("clusterIP")).clusterIP) |
			del(.. | select(has("clusterIPs")).clusterIPs) |
			del(.. | select(has("dataSource")).dataSource) |
			del(.. | select(has("procMount")).procMount) |
			del(.. | select(has("storageClassName")).storageClassName) |
			del(.. | select(has("finalizers")).finalizers) |
			del(.. | select(has("kubernetes.io/pvc-protection"))."kubernetes.io/pvc-protection") |
			del(.. | select(has("volumeName")).volumeName) |
			del(.. | select(has("volume.beta.kubernetes.io/storage-provisioner"))."volume.beta.kubernetes.io/storage-provisioner") |
			del(.. | select(has("volume.kubernetes.io/storage-provisioner"))."volume.kubernetes.io/storage-provisioner") |
			del(.spec.volumeMode) |
			del(.. | select(has("volume.kubernetes.io/selected-node"))."volume.kubernetes.io/selected-node") |
			del(.. | select(has("percona.com/last-config-hash"))."percona.com/last-config-hash") |
			del(.. | select(has("percona.com/configuration-hash"))."percona.com/configuration-hash") |
			del(.. | select(has("percona.com/ssl-hash"))."percona.com/ssl-hash") |
			del(.. | select(has("percona.com/ssl-internal-hash"))."percona.com/ssl-internal-hash") |
			del(.spec.volumeClaimTemplates[].spec.volumeMode | select(. == "Filesystem")) |
			del(.. | select(has("healthCheckNodePort")).healthCheckNodePort) |
			del(.. | select(has("nodePort")).nodePort) |
			del(.status) |
			(.. | select(tag == "!!str")) |= sub("'$namespace'", "NAME_SPACE") |
			del(.spec.volumeClaimTemplates[].apiVersion) |
			del(.spec.volumeClaimTemplates[].kind) |
			del(.spec.ipFamilies) |
			del(.spec.ipFamilyPolicy) |
			(.. | select(. == "extensions/v1beta1")) = "apps/v1" |
			(.. | select(. == "batch/v1beta1")) = "batch/v1" ' - >${new_result}

	yq -i eval 'del(.spec.persistentVolumeClaimRetentionPolicy)' ${new_result}

	if version_gt "1.22"; then
		yq -i eval 'del(.spec.internalTrafficPolicy)' ${new_result}
		yq -i eval 'del(.spec.allocateLoadBalancerNodePorts)' ${new_result}
		if [[ ${expected_result} == */cronjob* ]]; then
			yq -i eval 'del(.metadata.generation)' ${new_result}
		fi
	fi
	if [ -n "$skip_generation_check" ]; then
		yq -i eval 'del(.metadata.generation)' ${new_result}
	fi

	if [[ ${UPDATE_COMPARE_FILES} -eq 0 ]]; then
		if ! diff -u "$expected_result" "$new_result"; then
			log "compare_kubectl ($(caller)) failed with diff"
			exit 1
		fi
	else
		cp ${new_result} ${expected_result}
	fi

	log "compare_kubectl: ${resource} OK"
}

run_recovery_check() {
	local backup_name=$1
	local compare_suffix=${2:-"_restore"}
	local is_sharded=${3:-""}
	local tls=${4:-""}

	wait_restore "${backup_name}" "${cluster}" "requested" "0" "3000"
	echo

	compare_kubectl "statefulset/${cluster}-rs0" ${compare_suffix}

	# we don't wait for cluster readiness here because the annotation gets removed then
	wait_restore "${backup_name}" "${cluster}" "ready" "0" "3000"

	log "Physical restore finished: OK"

	if [ $(kubectl_bin get psmdb ${cluster} -o yaml | yq '.metadata.annotations."percona.com/resync-pbm"') == null ]; then
		echo "psmdb/${cluster} should be annotated with percona.com/resync-pbm after a physical restore"
		exit 1
	fi
	echo

	wait_cluster_consistency ${cluster}
	wait_for_pbm_operations ${cluster}

	log "Operator triggered PBM resync: OK"

	if [ -n "$is_sharded" ]; then
		compare_mongos_cmd "find" "myApp:myPass@${cluster}-mongos.${namespace}" "-sharded" "" "" "" "" "$tls"
	else
		compare_mongo_cmd "find" "myApp:myPass@${cluster}-rs0-0.${cluster}-rs0.${namespace}" "" "" "" "" "" "$tls"
		compare_mongo_cmd "find" "myApp:myPass@${cluster}-rs0-1.${cluster}-rs0.${namespace}" "" "" "" "" "" "$tls"
		compare_mongo_cmd "find" "myApp:myPass@${cluster}-rs0-2.${cluster}-rs0.${namespace}" "" "" "" "" "" "$tls"
	fi

	log "Data restored: OK"

	echo "${cluster}-rs0-0:/data/db/pbm-restore-logs"
	kubectl_bin exec ${cluster}-rs0-0 -- ls -l /data/db/pbm-restore-logs
	echo "${cluster}-rs0-1:/data/db/pbm-restore-logs"
	kubectl_bin exec ${cluster}-rs0-1 -- ls -l /data/db/pbm-restore-logs
	echo "${cluster}-rs0-2:/data/db/pbm-restore-logs"
	kubectl_bin exec ${cluster}-rs0-2 -- ls -l /data/db/pbm-restore-logs
	if [ -n "$is_sharded" ]; then
		echo "${cluster}-cfg-0:/data/db/pbm-restore-logs"
		kubectl_bin exec ${cluster}-cfg-0 -- ls -l /data/db/pbm-restore-logs
		echo "${cluster}-cfg-1:/data/db/pbm-restore-logs"
		kubectl_bin exec ${cluster}-cfg-1 -- ls -l /data/db/pbm-restore-logs
		echo "${cluster}-cfg-2:/data/db/pbm-restore-logs"
		kubectl_bin exec ${cluster}-cfg-2 -- ls -l /data/db/pbm-restore-logs
	fi

	log "Restore logs are preserved: OK"
}

run_mongo() {
	local command="$1"
	local uri="$2"
	local driver=${3:-mongodb+srv}
	local suffix=${4:-.svc.cluster.local}
	local mongo_flag="$5"
	local replica_set=${6:-"rs0"}
	if [[ $uri == *cfg* ]]; then
		replica_set="cfg"
	fi

	local client_container=$(kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}')
	kubectl_bin exec ${client_container} -- \
		bash -c "printf '$command\n' | mongo $driver://$uri$suffix/admin?ssl=false\&replicaSet=$replica_set $mongo_flag"
}

run_mongosh() {
	local command="$1"
	local uri="$2"
	local driver=${3:-"mongodb+srv"}
	local suffix=${4:-".svc.cluster.local"}
	local mongo_flag="$5"

	local client_container=$(kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}')
	[[ $uri == *cfg* ]] && replica_set='cfg' || replica_set='rs0'
	kubectl_bin exec ${client_container} -- \
		bash -c "printf '$command\n' | mongosh --quiet $driver://$uri$suffix/admin?ssl=false\&replicaSet=$replica_set $mongo_flag"
}

run_mongo_tls() {
	local command="$1"
	local uri="$2"
	local driver=${3:-mongodb+srv}
	local suffix=${4:-.svc.cluster.local}
	local client_container=$(kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}')
	local mongo_flag="$5"
	local port=${6:-"27017"}

	suffix_port=$(echo "$suffix" | awk -F':' '{print $2}')
	if [[ -z $suffix_port ]]; then
		suffix="$suffix:$port"
	fi

	[[ $uri == *cfg* ]] && replica_set='cfg' || replica_set='rs0'
	kubectl_bin exec ${client_container} -- \
		bash -c "printf '$command\n' | mongo $driver://$uri$suffix/admin?replicaSet=$replica_set --tls --tlsCAFile /etc/mongodb-ssl/ca.crt  --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames $mongo_flag"

}

run_mongos() {
	local command="$1"
	local uri="$2"
	local driver=${3:-mongodb}
	local suffix=${4:-.svc.cluster.local}
	local mongo_flag="$5"
	local port=${6:-"27017"}
	local mongo_bin=${7:-mongo}

	suffix_port=$(echo "$suffix" | awk -F':' '{print $2}')
	if [[ -z $suffix_port ]]; then
		suffix="$suffix:$port"
	fi

	local client_container=$(kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}')

	kubectl_bin exec ${client_container} -- \
		bash -c "printf '$command\n' | ${mongo_bin} $driver://$uri$suffix/admin $mongo_flag"
}

run_mongos_tls() {
	local command="$1"
	local uri="$2"
	local driver=${3:-mongodb}
	local suffix=${4:-.svc.cluster.local}
	local mongo_flag="$5"
	local port=${6:-"27017"}
	local mongo_bin=${7:-mongo}

	suffix_port=$(echo "$suffix" | awk -F':' '{print $2}')
	if [[ -z $suffix_port ]]; then
		suffix="$suffix:$port"
	fi

	local client_container=$(kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}')

	kubectl_bin exec ${client_container} -- \
		bash -c "printf '$command\n' | ${mongo_bin} $driver://$uri$suffix/admin --tls --tlsCAFile /etc/mongodb-ssl/ca.crt  --tlsCertificateKeyFile /tmp/tls.pem --tlsAllowInvalidHostnames $mongo_flag"
}

run_script_mongos() {
	local script="$1"
	local uri="$2"
	local driver=${3:-mongodb}
	local suffix=${4:-.svc.cluster.local}
	local mongo_flag="$5"
	local mongo_bin=${6:-mongo}

	local client_container=$(kubectl_bin get pods --selector=name=psmdb-client -o 'jsonpath={.items[].metadata.name}')

	name="$(basename $script)"
	kubectl_bin cp ${script} $namespace/${client_container}:/tmp
	kubectl_bin exec ${client_container} -- \
		bash -c "${mongo_bin} $driver://$uri$suffix/admin $mongo_flag /tmp/${name}"
}

get_pmm_service_ip() {
	local service=$1

	while (kubectl_bin get service/$service -o 'jsonpath={.spec.type}' 2>&1 || :) | grep -q NotFound; do
		sleep 1
	done
	until kubectl_bin get service/$service -o 'jsonpath={.status.loadBalancer.ingress[]}' 2>&1 | grep -E -q "hostname|ip"; do
		sleep 1
	done
	kubectl_bin get service/$service -o 'jsonpath={.status.loadBalancer.ingress[].ip}'
	kubectl_bin get service/$service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}'
}

get_service_ip() {
	local service=$1
	local server_type=${2:-rs0}

	if [ "$(kubectl_bin get psmdb/${service/-$server_type*/} -o 'jsonpath={.spec.replsets[].expose.enabled}')" != "true" ]; then
		echo -n $service.${service/-rs0*/}-rs0
		return
	fi
	while (kubectl_bin get service/$service -o 'jsonpath={.spec.type}' 2>&1 || :) | grep -q NotFound; do
		sleep 1
	done
	service_type=$(kubectl_bin get service/"$service" -o 'jsonpath={.spec.type}')
	if [ "$service_type" = "ClusterIP" ] || [ "$service_type" = "NodePort" ]; then
		kubectl_bin get service/$service -o 'jsonpath={.spec.clusterIP}'
		return
	fi
	until kubectl_bin get service/$service -o 'jsonpath={.status.loadBalancer.ingress[]}' 2>&1 | grep -E -q "hostname|ip"; do
		sleep 1
	done
	kubectl_bin get service/$service -o 'jsonpath={.status.loadBalancer.ingress[].ip}'
	kubectl_bin get service/$service -o 'jsonpath={.status.loadBalancer.ingress[].hostname}'
}

compare_mongo_cmd() {
	local command="$1"
	local uri="$2"
	local postfix="$3"
	local suffix="$4"
	local database="${5:-myApp}"
	local collection="${6:-test}"
	local sort="$7"
	local tls="${8:-false}"
	local replicaset="$9"

	local full_command="db.${collection}.${command}()"
	if [[ -n ${sort} ]]; then
		full_command="${full_command}.${sort}"
	fi

	log "running ${full_command} in ${database}"

	if [[ $tls == "true" ]]; then
		mongo_command=run_mongo_tls
	else
		mongo_command=run_mongo
	fi

	$mongo_command "use ${database}\n ${full_command}" "$uri" "mongodb" "$suffix" "" ${replicaset} \
		| grep -E -v 'I NETWORK|W NETWORK|F NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' \
		| $sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' \
			>$tmp_dir/${command}${postfix}

	if [[ ${UPDATE_COMPARE_FILES} -eq 0 ]]; then
		diff -u ${test_dir}/compare/${command}${postfix}.json $tmp_dir/${command}${postfix}
	else
		cp $tmp_dir/${command}${postfix} ${test_dir}/compare/${command}${postfix}.json
	fi
}

compare_mongos_cmd() {
	local command="$1"
	local uri="$2"
	local postfix="$3"
	local suffix="$4"
	local database="${5:-myApp}"
	local collection="${6:-test}"
	local port="${7:-27017}"
	local tls="${8:-false}"

	if [[ $tls == "true" ]]; then
		mongos_command=run_mongos_tls
	else
		mongos_command=run_mongos
	fi

	log "running db.${collection}.command() in ${database}"

	$mongos_command "use ${database}\n db.${collection}.${command}()" "$uri" "mongodb" "$suffix" "" "$port" \
		| grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|Started a new thread for the timer service' \
		| $sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/' \
			>$tmp_dir/${command}${postfix}

	if [[ ${UPDATE_COMPARE_FILES} -eq 0 ]]; then
		diff ${test_dir}/compare/${command}${postfix}.json $tmp_dir/${command}${postfix}
	else
		cp $tmp_dir/${command}${postfix} ${test_dir}/compare/${command}${postfix}.json
	fi
}

get_mongo_primary_endpoint() {
	local uri="$1"

	run_mongo 'db.hello().me' "$uri" "mongodb" ":27017" \
		| grep -E -v "Time|Percona Server for MongoDB|bye|BinData|NumberLong|connecting to|Error saving history file|I NETWORK|W NETWORK|Implicit session:|versions do not match" \
		| sed -e 's^20[0-9][0-9]-[0-9][0-9]-[0-9][0-9]T[0-9][0-9]:[0-9][0-9]:[0-9][0-9]\.[0-9][0-9][0-9]+[0-9][0-9][0-9][0-9]^^' \
		| grep ":27017$"
}

get_mongo_primary() {
	local uri="$1"
	local cluster="$2"

	set_debug
	endpoint=$(get_mongo_primary_endpoint $uri)
	if [[ $endpoint =~ ".$cluster" ]]; then
		echo $endpoint \
			| cut -d . -f 1
	else
		kubectl_bin get service -o wide \
			| grep " ${endpoint/:*/} " \
			| awk '{print $1}'
	fi
}

check_exported_mongos_service_endpoint() {
	local host=$1

	if [ "$host" != "$(kubectl_bin get psmdb $cluster -o=jsonpath='{.status.host}')" ]; then
		echo "Exported host is not correct after the restore"
		exit 1
	fi
}

compare_mongo_user() {
	local uri="$1"
	local user="$2"
	local expected_result=${test_dir}/compare/$user.json

	if [[ $IMAGE_MONGOD =~ 4\.0 ]] && [ -f ${test_dir}/compare/$user-40.json ]; then
		expected_result=${test_dir}/compare/$user-40.json
	fi
	if [[ $IMAGE_MONGOD =~ 4\.2 ]] && [ -f ${test_dir}/compare/$user-42.json ]; then
		expected_result=${test_dir}/compare/$user-42.json
	fi
	if [[ $IMAGE_MONGOD =~ 4\.4 ]] && [ -f ${test_dir}/compare/$user-44.json ]; then
		expected_result=${test_dir}/compare/$user-44.json
	fi
	if [[ $IMAGE_MONGOD =~ 5\.0 ]] && [ -f ${test_dir}/compare/$user-50.json ]; then
		expected_result=${test_dir}/compare/$user-50.json
	fi
	if [[ $IMAGE_MONGOD =~ 6\.0 ]] && [ -f ${test_dir}/compare/$user-60.json ]; then
		expected_result=${test_dir}/compare/$user-60.json
	fi
	if [[ $IMAGE_MONGOD =~ 7\.0 ]] && [ -f ${test_dir}/compare/$user-70.json ]; then
		expected_result=${test_dir}/compare/$user-70.json
	fi
	if [[ $IMAGE_MONGOD =~ 8\.0 ]] && [ -f ${test_dir}/compare/$user-80.json ]; then
		expected_result=${test_dir}/compare/$user-80.json
	fi

	run_mongo 'db.runCommand({connectionStatus:1,showPrivileges:true})' "$uri" \
		| grep -E -v "Time|Percona Server for MongoDB|bye|BinData|NumberLong|connecting to|Error saving history file|I NETWORK|W NETWORK|Implicit session:|versions do not match" \
		| sed -e 's^20[0-9][0-9]-[0-9][0-9]-[0-9][0-9]T[0-9][0-9]:[0-9][0-9]:[0-9][0-9]\.[0-9][0-9][0-9]+[0-9][0-9][0-9][0-9]^^' \
		| $sed -e '/"ok" : 1/,+4d' \
		| sed -e '$s/,$/}/' \
		| jq '.authInfo.authenticatedUserPrivileges|=sort_by(.resource.anyResource, .resource.cluster, .resource.db, .resource.collection)|.authInfo.authenticatedUserRoles|=sort_by(.role)' \
			>$tmp_dir/$user.json

	if [[ ${UPDATE_COMPARE_FILES} -eq 0 ]]; then
		diff -u $expected_result $tmp_dir/$user.json
	else
		cp $tmp_dir/$user.json $expected_result
	fi
}

start_gke() {
	gcloud container clusters create operator-testing-$RANDOM --zone europe-west3-c --project cloud-dev-112233 --preemptible --cluster-version 1.11
}

get_pumba() {
	kubectl_bin get pods \
		--selector=name=pumba \
		-o 'jsonpath={.items[].metadata.name}'
}

run_pumba() {
	local cmd="$*"
	kubectl_bin exec -it "$(get_pumba)" -- /pumba -l info ${cmd}
}

deploy_cert_manager() {
	desc 'deploy cert manager'

	kubectl_bin create namespace cert-manager || :
	kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true || :
	kubectl_bin apply -f "https://github.com/cert-manager/cert-manager/releases/download/v${CERT_MANAGER_VER}/cert-manager.yaml" --validate=false || : 2>/dev/null
	for arg in "$@"; do
		kubectl_bin patch deployment cert-manager -n cert-manager --type='json' \
			-p='[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"'"$arg"'"}]'
	done
	kubectl_bin -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready
	sleep 120
}

delete_crd() {
	desc 'get and delete old CRDs and RBAC'

	kubectl_bin delete -f "${src_dir}/deploy/crd.yaml" --ignore-not-found --wait=false || :
	for crd_name in $(yq eval '.metadata.name' "${src_dir}/deploy/crd.yaml" | grep -v '\-\-\-'); do
		kubectl get ${crd_name} --all-namespaces -o wide \
			| grep -v 'NAMESPACE' \
			| xargs -L 1 sh -xc 'kubectl patch '${crd_name}' -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' \
			|| :
		kubectl_bin wait --for=delete crd ${crd_name} || :
	done

	local rbac_yaml='rbac.yaml'
	if [ -n "${OPERATOR_NS}" ]; then
		rbac_yaml='cw-rbac.yaml'
	fi

	kubectl_bin delete -f "${src_dir}/deploy/$rbac_yaml" --ignore-not-found || true
}

delete_backups() {
	desc 'Delete psmdb-backup'
	if [ $(kubectl_bin get psmdb-backup --no-headers | wc -l) != 0 ]; then
		kubectl_bin get psmdb-backup
		kubectl_bin delete psmdb-backup --all
	fi
}

destroy() {
	local namespace="$1"
	local ignore_logs="${2:-true}"

	if [[ ${SKIP_DELETE} == 1 ]]; then
		echo "SKIP_DELETE=1, not destroying ${namespace}"
		return
	fi

	desc 'destroy cluster/operator and all other resources'
	if [ ${ignore_logs} == "false" ] && [ "${DEBUG_TESTS}" == 1 ]; then
		kubectl_bin logs ${OPERATOR_NS:+-n $OPERATOR_NS} $(get_operator_pod) \
			| grep -v 'level=info' \
			| grep -v 'level=debug' \
			| grep -v 'Getting tasks for pod' \
			| grep -v 'Getting pods from source' \
			| grep -v 'the object has been modified' \
			| grep -v 'get backup status: Job.batch' \
			| $sed -r 's/"ts":[0-9.]+//; s^limits-[0-9.]+/^^g' \
			| sort -u \
			| tee $tmp_dir/operator.log
	fi
	#TODO: maybe will be enabled later
	#diff $test_dir/compare/operator.log $tmp_dir/operator.log

	delete_backups

	delete_crd

	destroy_cert_manager || true
	if [ -n "$OPENSHIFT" ]; then
		oc delete --grace-period=0 --force=true project "$namespace" &
		if [ -n "$OPERATOR_NS" ]; then
			oc delete --grace-period=0 --force=true project "$OPERATOR_NS" &
		fi
	else
		kubectl_bin delete --grace-period=0 --force=true namespace "$namespace" &
		if [ -n "$OPERATOR_NS" ]; then
			kubectl_bin delete --grace-period=0 --force=true namespace "$OPERATOR_NS" &
		fi
	fi
	rm -rf ${tmp_dir}
}

destroy_cert_manager() {
	kubectl_bin delete -f "https://github.com/cert-manager/cert-manager/releases/download/v${CERT_MANAGER_VER}/cert-manager.yaml"
}

desc() {
	set +o xtrace
	local msg="$@"
	printf "\n\n-----------------------------------------------------------------------------------\n"
	printf "$msg"
	printf "\n-----------------------------------------------------------------------------------\n\n"
	set_debug
}

get_backup_dest() {
	local backup_name=$1

	kubectl_bin get psmdb-backup $backup_name -o jsonpath='{.status.destination}' \
		| sed -e 's/.json$//' | sed "s|s3://||" | sed "s|azure://||" | sed "s|gs://||"
}

get_service_endpoint() {
	local service=$1

	local hostname=$(
		kubectl_bin get service/$service -o json \
			| jq '.status.loadBalancer.ingress[].hostname' \
			| sed -e 's/^"//; s/"$//;'
	)
	if [ -n "$hostname" -a "$hostname" != "null" ]; then
		echo $hostname
		return
	fi

	local ip=$(
		kubectl_bin get service/$service -o json \
			| jq '.status.loadBalancer.ingress[].ip' \
			| sed -e 's/^"//; s/"$//;'
	)
	if [ -n "$ip" -a "$ip" != "null" ]; then
		echo $ip
		return
	fi

	exit 1
}

get_metric_values() {
	local metric=$1
	local instance=$2
	local user_pass=$3
	local start=$($date -u "+%s" -d "-1 minute")
	local end=$($date -u "+%s")
	local endpoint=$(get_service_endpoint monitoring-service)

	curl -s -k "https://${user_pass}@$endpoint/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28$metric%7Bnode_name%3D%7E%22$instance%22%7d%20or%20$metric%7Bnode_name%3D%7E%22$instance%22%7D%29&start=$start&end=$end&step=60" \
		| jq '.data.result[0].values[][1]' \
		| grep '^"[0-9]'

}

get_qan_values() {
	local service_type=$1
	local environment=$2
	local user_pass=$3
	local start
	local end
	local endpoint
	start=$($date -u -d '-12 hour' '+%Y-%m-%dT%H:%M:%S%:z')
	end=$($date -u '+%Y-%m-%dT%H:%M:%S%:z')
	endpoint=$(get_service_endpoint monitoring-service)

	cat >payload.json <<EOF
{
  "period_start_from": "$start",
  "period_start_to": "$end",
  "group_by": "queryid",
  "labels": [
    {
      "key": "service_type",
      "value": [
        "$service_type"
      ]
    },
    {
      "key": "environment",
      "value": [
        "$environment"
      ]
    }
  ],
  "columns": [
    "load",
    "num_queries",
    "query_time"
  ],
  "order_by": "-load",
  "offset": 0,
  "limit": 10,
  "main_metric": "load",
  "search": ""
}
EOF

	local response

	retry=0

	until [[ "$(curl -s -k -XPOST -d @payload.json "https://${user_pass}@$endpoint/v0/qan/GetReport" \
		| jq '.rows[].sparkline')" != "null" ]]; do
		let retry+=1
		if [ $retry -ge 10 ]; then
			echo max retry count $retry reached. "No data for $service_type service type in QAN"
			exit 1
		fi
		echo -n .
		sleep 5
	done
	rm -f payload.json
}

cat_config() {
	cat "$1" \
		| yq eval '(.spec | select(.image == null)).image = "'"$IMAGE_MONGOD"'"' \
		| yq eval '(.spec | select(has("pmm"))).pmm.image = "'"$IMAGE_PMM_CLIENT"'"' \
		| yq eval '(.spec | select(has("initImage"))).initImage = "'"$IMAGE"'"' \
		| yq eval '(.spec | select(has("backup"))).backup.image = "'"$IMAGE_BACKUP"'"' \
		| $sed -e "s/NAME_SPACE/${namespace}/g" \
		| yq eval '.spec.upgradeOptions.apply="Never"'
}

format_date() {
	local timestamp=$1
	echo $(TZ=UTC $date -d@${timestamp} '+%Y-%m-%d %H:%M:%S')
}

apply_cluster() {
	if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then
		cat_config "$1" \
			| kubectl_bin apply -f -
	else
		cat_config "$1" \
			| yq eval '
				del(.spec.backup.tasks.[1]) |
				del(.spec.backup.tasks.[1]) |
				del(.spec.backup.tasks.[1])' - \
			| kubectl_bin apply -f -
	fi
}

# get_cluster_gh echoes the default cr.yaml on github
# so caller can do intended modifications and apply it
get_cluster_gh() {
	local git_tag=$1

	curl -s "https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/cr.yaml"
}

spinup_psmdb() {
	local cluster=$1
	local config=$2
	local size="${3:-3}"

	desc 'create first PSMDB cluster'
	apply_cluster $config

	desc 'check if Pod is started'
	wait_for_running "${cluster}" "$size"
	sleep 20

	compare_kubectl "statefulset/${cluster}"

	desc 'write data'

	run_mongo 'db.createUser({user: "myApp", pwd: "myPass", roles: [{ db: "myApp", role: "readWrite" }]})' \
		"userAdmin:userAdmin123456@${cluster}.${namespace}"

	run_mongo 'use myApp\n db.test.insert({ x: 100500 })' "myApp:myPass@${cluster}.${namespace}"
}

kubectl_bin() {
	local LAST_OUT="$(mktemp)"
	local LAST_ERR="$(mktemp)"
	local exit_status=0
	local timeout=4

	for i in $(seq 0 2); do
		set +e
		kubectl "$@" 1>"$LAST_OUT" 2>"$LAST_ERR"
		exit_status=$?
		set -e
		if [ ${exit_status} != 0 -a -n "${DEBUG_TESTS}" ]; then
			cat "$LAST_OUT"
			cat "$LAST_ERR" >&2
			sleep "$((timeout * i))"
		else
			break
		fi
	done

	cat "$LAST_OUT"
	cat "$LAST_ERR" >&2
	rm "$LAST_OUT" "$LAST_ERR"
	return ${exit_status}
}

patch_secret() {
	local secret=$1
	local key=$2
	local value=$3

	kubectl patch secret $secret -p="{\"data\":{\"$key\": \"$value\"}}"
}

getSecretData() {
	local secretName=$1
	local dataKey=$2
	local data=$(kubectl get secrets/${secretName} --template={{.data.${dataKey}}} | base64 -d)
	echo "$data"
}

check_mongo_auth() {
	local uri="$1"

	ping=$(run_mongo "db.runCommand({ ping: 1 }).ok" "$uri" "mongodb" "" "--quiet" | grep -E -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:')
	desc "ping return"
	if [ "${ping}" != "1" ]; then
		return 1
	fi
}

wait_cluster_consistency() {
	local cluster_name=$1
	local wait_time=${2:-32}

	retry=0
	sleep 7 # wait for two reconcile loops ;)  3 sec x 2 times + 1 sec = 7 seconds
	echo -n 'waiting for cluster readyness'
	until [[ "$(kubectl_bin get psmdb "${cluster_name}" -o jsonpath='{.status.state}')" == "ready" ]]; do
		let retry+=1
		if [ $retry -ge $wait_time ]; then
			echo max retry count $retry reached. something went wrong with operator or kubernetes cluster
			exit 1
		fi
		echo -n .
		sleep 10
	done
	echo '.OK'
}

run_backup() {
	local storage=$1
	local backup_name=${2:-"backup-${storage}"}
	local type=${3:-"logical"}

	log "running backup $backup_name"

	yq eval '.metadata.name = "'${backup_name}'"
			| .spec.storageName = "'${storage}'"
			| .spec.type = "'${type}'"' \
		$test_dir/conf/backup-$storage.yml \
		| kubectl_bin apply -f -
}

function check_backup_deletion_aws() {
	bucket=$(echo "$1" | cut -d'/' -f1)
	key_prefix=$(echo "$1" | cut -d'/' -f2-)
	key=$2
	storage_name="aws-s3"
	retry=0

	while aws s3api head-object --bucket "$bucket" --key "${key_prefix}${key}" &>/dev/null; do
		if [ $retry -ge 15 ]; then
			echo "max retry count $retry reached. something went wrong with operator or kubernetes cluster"
			echo "Backup still exists in $storage_name (expected it to be deleted)"
			exit 1
		fi
		echo "waiting for backup to be deleted from $storage_name"
		sleep 10
		((retry += 1))
	done

	echo "Backup ${key_prefix}${key} in bucket $bucket not found in $storage_name"
}

function check_backup_deletion_gcs() {
	backup_dest_gcp=$1
	obj=$2
	storage_name="gcp-cs"
	retry=0
	gcs_path="gs://${backup_dest_gcp}${obj}"

	while gsutil ls "$gcs_path" >/dev/null 2>&1; do
		if [ $retry -ge 15 ]; then
			echo "max retry count $retry reached. something went wrong with operator or kubernetes cluster"
			echo "Backup $gcs_path still exists in $storage_name (expected it to be deleted)"
			exit 1
		fi
		echo "waiting for backup to be deleted from $storage_name"
		sleep 10
		((retry += 1))
	done

	echo "Backup $gcs_path not found in $storage_name"
}

function check_backup_deletion_azure() {
	url_path=$(echo "$1" | sed 's|https://[^/]*\.blob\.core\.windows\.net/||')
	container=$(echo "$url_path" | cut -d'/' -f1)
	blob_prefix=$(echo "$url_path" | cut -d'/' -f2-)

	blob=$2
	storage_name="azure-blob"
	retry=0
	blob_path="${blob_prefix}${blob}"

	while az storage blob show --container-name "$container" --name "$blob_path" &>/dev/null; do
		if [ $retry -ge 15 ]; then
			echo "max retry count $retry reached. something went wrong with operator or kubernetes cluster"
			echo "Backup still exists in $storage_name (expected it to be deleted)"
			exit 1
		fi
		echo "waiting for backup to be deleted from $storage_name"
		sleep 10
		((retry += 1))
	done

	echo "Backup ${blob_path} in container $container not found in $storage_name"
}

check_backup_deletion() {
	local path=$1
	local storage_name=$2
	local retry=0

	until [[ $(curl -sw '%{http_code}' -o /dev/null $path) -eq 403 ]] || [[ $(curl -sw '%{http_code}' -o /dev/null $path) -eq 404 ]]; do
		if [ $retry -ge 10 ]; then
			echo max retry count $retry reached. something went wrong with operator or kubernetes cluster
			echo "Backup was not removed from bucket -- $storage_name"
			exit 1
		fi
		echo "waiting for backup deletion $storage_name"
		sleep $((10 * 2 ** retry))
		((retry += 1))
	done
}

function setup_aws_credentials() {
	local secret_name="aws-s3-secret"

	if [[ -n $AWS_ACCESS_KEY_ID ]] && [[ -n $AWS_SECRET_ACCESS_KEY ]]; then
		echo "AWS credentials already set in environment"
		return 0
	fi

	echo "Setting up AWS credentials from secret: $secret_name"

	# Disable tracing for the entire credential section
	local trace_was_on=0
	if [[ $- == *x* ]]; then
		trace_was_on=1
		set +x
	fi

	AWS_ACCESS_KEY_ID=$(kubectl get secret "$secret_name" -o jsonpath='{.data.AWS_ACCESS_KEY_ID}' 2>/dev/null | base64 -d 2>/dev/null)
	AWS_SECRET_ACCESS_KEY=$(kubectl get secret "$secret_name" -o jsonpath='{.data.AWS_SECRET_ACCESS_KEY}' 2>/dev/null | base64 -d 2>/dev/null)

	if [[ -z $AWS_ACCESS_KEY_ID ]] || [[ -z $AWS_SECRET_ACCESS_KEY ]]; then
		# Re-enable tracing before error message if it was on
		[[ $trace_was_on -eq 1 ]] && set -x
		echo "Failed to extract AWS credentials from secret"
		return 1
	fi

	export AWS_ACCESS_KEY_ID
	export AWS_SECRET_ACCESS_KEY

	# Re-enable tracing if it was on
	[[ $trace_was_on -eq 1 ]] && set -x

	echo "AWS credentials configured successfully"
}

function setup_gcs_credentials() {
	local secret_name="gcp-cs-secret"

	if gsutil ls >/dev/null 2>&1; then
		echo "GCS credentials already set in environment"
		return 0
	fi

	echo "Setting up GCS credentials from K8s secret: $secret_name"

	# Disable tracing for the entire credential section
	local trace_was_on=0
	if [[ $- == *x* ]]; then
		trace_was_on=1
		set +x
	fi

	ACCESS_KEY_ID=$(kubectl get secret "$secret_name" -o jsonpath='{.data.AWS_ACCESS_KEY_ID}' 2>/dev/null | base64 -d 2>/dev/null)
	SECRET_ACCESS_KEY=$(kubectl get secret "$secret_name" -o jsonpath='{.data.AWS_SECRET_ACCESS_KEY}' 2>/dev/null | base64 -d 2>/dev/null)

	if [[ -z $ACCESS_KEY_ID ]] || [[ -z $SECRET_ACCESS_KEY ]]; then
		# Re-enable tracing before error message if it was on
		[[ $trace_was_on -eq 1 ]] && set -x
		echo "Failed to extract GCS credentials from secret" >&2
		return 1
	fi

	boto_tmp=$(mktemp /tmp/boto.XXXXXX)
	chmod 600 "$boto_tmp"

	cat <<EOF >"$boto_tmp"
[Credentials]
gs_access_key_id = ${ACCESS_KEY_ID}
gs_secret_access_key = ${SECRET_ACCESS_KEY}
EOF

	export BOTO_CONFIG="$boto_tmp"

	unset ACCESS_KEY_ID
	unset SECRET_ACCESS_KEY

	[[ $trace_was_on -eq 1 ]] && set -x

	echo "GCS credentials configured successfully"
}

function setup_azure_credentials() {
	local secret_name="azure-secret"

	echo "Setting up Azure credentials from K8s secret: $secret_name"

	# Disable tracing for the entire credential section
	local trace_was_on=0
	if [[ $- == *x* ]]; then
		trace_was_on=1
		set +x
	fi

	AZURE_STORAGE_ACCOUNT=$(kubectl_bin get secret "$secret_name" -o jsonpath='{.data.AZURE_STORAGE_ACCOUNT_NAME}' 2>/dev/null | base64 -d 2>/dev/null)
	AZURE_STORAGE_KEY=$(kubectl_bin get secret "$secret_name" -o jsonpath='{.data.AZURE_STORAGE_ACCOUNT_KEY}' 2>/dev/null | base64 -d 2>/dev/null)

	if [[ -z $AZURE_STORAGE_ACCOUNT ]] || [[ -z $AZURE_STORAGE_KEY ]]; then
		# Re-enable tracing before error message if it was on
		[[ $trace_was_on -eq 1 ]] && set -x
		echo "Failed to extract Azure credentials from secret" >&2
		return 1
	fi

	export AZURE_STORAGE_ACCOUNT
	export AZURE_STORAGE_KEY

	# Re-enable tracing if it was on
	[[ $trace_was_on -eq 1 ]] && set -x

	echo "Azure credentials configured successfully"
}

create_infra() {
	local ns="$1"

	if [[ ${DELETE_CRD_ON_START} == 1 ]]; then
		delete_crd
		check_crd_for_deletion "${GIT_BRANCH}"
	fi
	if [ -n "$OPERATOR_NS" ]; then
		create_namespace $OPERATOR_NS
		deploy_operator
		create_namespace $ns
	else
		create_namespace $ns
		deploy_operator
	fi
}

create_infra_gh_release() {
	local ns="$1"
	local git_tag="$2"

	check_crd_for_deletion "${git_tag}"
	if [ -n "${OPERATOR_NS}" ]; then
		create_namespace "${OPERATOR_NS}"
		deploy_operator_gh_release "${git_tag}"
		create_namespace "${ns}"
	else
		create_namespace "${ns}"
		deploy_operator_gh_release "${git_tag}"
	fi
}

create_infra_gh() {
	local ns="$1"
	local git_tag="$2"

	check_crd_for_deletion "${git_tag}"
	if [ -n "${OPERATOR_NS}" ]; then
		create_namespace "${OPERATOR_NS}"
		deploy_operator_gh "${git_tag}"
		create_namespace "${ns}"
	else
		create_namespace "${ns}"
		deploy_operator_gh "${git_tag}"
	fi
}

check_crd_for_deletion() {
	# 1.24 rejects CR object creation if the corresponding CRD object is to be removed.
	# We need to make sure that it will happen before the next CR creation request.
	local git_tag="$1"

	for crd_name in $(curl -s https://raw.githubusercontent.com/percona/percona-server-mongodb-operator/${git_tag}/deploy/crd.yaml | yq eval '.metadata.name' | $sed 's/---//g' | $sed ':a;N;$!ba;s/\n/ /g'); do
		if [[ $(kubectl_bin get crd/${crd_name} -o jsonpath='{.status.conditions[-1].type}') == "Terminating" ]]; then
			kubectl get ${crd_name} --all-namespaces -o wide \
				| grep -v 'NAMESPACE' \
				| xargs -L 1 sh -xc 'kubectl patch '${crd_name}' -n $0 $1 --type=merge -p "{\"metadata\":{\"finalizers\":[]}}"' \
				|| :
		fi
	done
}

function get_mongod_ver_from_image() {
	local image=${1}

	version_info=$(run_simple_cli_inside_image ${image} 'mongod --version' | $sed -r 's/^.*db version v(([0-9]+\.){2}[0-9]+-[0-9]+).*$/\1/g')

	if [[ ! ${version_info} =~ ^([0-9]+\.){2}[0-9]+-[0-9]+$ ]]; then
		printf "No mongod version obtained from %s. Exiting" ${image}
		exit 1
	fi
	echo ${version_info}
}

function get_pbm_version() {
	local image=${1}

	local version_info=$(run_simple_cli_inside_image ${image} 'pbm-agent version' | $sed -r 's/^Version:\ (([0-9]+\.){2}[0-9]+)\ .*/\1/g')

	if [[ ! ${version_info} =~ ^([0-9]+\.){2}[0-9]+$ ]]; then
		printf "No pbm version obtained from %s. Exiting" ${image}
		exit 1
	fi
	echo ${version_info}
}

function run_simple_cli_inside_image() {
	local image=${1}
	local cli=${2}

	local pod_name=${RANDOM}
	kubectl_bin -n default run ${pod_name} --image=${image} --restart=Never --command -- sleep infinity >/dev/null
	kubectl_bin -n default wait --for=condition=Ready pod/${pod_name} >/dev/null
	local output=$(kubectl_bin -n default exec ${pod_name} -- bash -c "${cli} 2>&1")
	kubectl_bin -n default delete pod/${pod_name} --grace-period=0 --force >/dev/null
	echo ${output}
}

function generate_vs_json() {
	local template_path=${1}
	local target_path=${2}

	local version_service_source=$(jq '.versions[0].operator="'${OPERATOR_VERSION}'"' ${template_path})

	for image_mongod in ${IMAGE_MONGOD_CHAIN[@]}; do
		current_mongod_version=$(get_mongod_ver_from_image ${image_mongod})

		version_service_source=$(echo ${version_service_source} \
			| jq '.versions[0].matrix.mongod += {"'${current_mongod_version}'": {"image_path":"'${image_mongod}'","status":"recommended"}}')
	done

	version_service_source=$(echo ${version_service_source} \
		| jq '.versions[0].matrix.backup += {"'$(get_pbm_version ${IMAGE_BACKUP})'": {"image_path":"'${IMAGE_BACKUP}'","status":"recommended"}}')

	version_service_source=$(echo ${version_service_source} \
		| jq '.versions[0].matrix.operator += {"'${OPERATOR_VERSION}'": {"image_path":"'${IMAGE}'","status":"recommended"}}')

	echo ${version_service_source} | jq '.' >${target_path}
}

check_passwords_leak() {
	local secrets
	local passwords
	local pods

	secrets=$(kubectl_bin get secrets -o json | jq -r '.items[].data | to_entries | .[] | select(.key | (contains("_PASSWORD"))) | .value')
	echo secrets=$secrets

	passwords="$(for i in $secrets; do
		base64 -d <<<$i
		echo
	done) $secrets"
	echo passwords=$passwords

	pods=$(kubectl_bin get pods -o name | awk -F "/" '{print $2}')
	echo pods=$pods

	collect_logs() {
		local containers
		local count

		NS=$1
		for p in $pods; do
			containers=$(kubectl_bin -n "$NS" get pod $p -o jsonpath='{.spec.containers[*].name}')
			for c in $containers; do
				# temporary, because of: https://jira.percona.com/browse/PMM-8357
				if [[ ${c} =~ "pmm" ]]; then
					continue
				fi
				kubectl_bin -n "$NS" logs $p -c $c >${tmp_dir}/logs_output-$p-$c.txt
				echo logs saved in: ${tmp_dir}/logs_output-$p-$c.txt
				for pass in $passwords; do
					count=$(grep -c --fixed-strings -- "$pass" ${tmp_dir}/logs_output-$p-$c.txt || :)
					if [[ $count != 0 ]]; then
						echo leaked passwords are found in log ${tmp_dir}/logs_output-$p-$c.txt
						false
					fi
				done
			done
			echo
		done
	}

	collect_logs $namespace
	if [ -n "$OPERATOR_NS" ]; then
		pods=$(kubectl_bin -n "${OPERATOR_NS}" get pods -o name | awk -F "/" '{print $2}')
		collect_logs $OPERATOR_NS
	fi
}

insert_data_mongos() {
	local data=$1
	local db_name=$2
	local flags=$3
	local port=${4:-"27017"}

	run_mongos \
		"use ${db_name}\n db.test.insert({ x: ${data} })" \
		"myApp:myPass@$cluster-mongos.$namespace" "" "" \
		"$flags" "$port"
}

deploy_pmm_server() {
	helm uninstall monitoring || :
	helm repo remove stable || :
	helm repo add stable https://charts.helm.sh/stable
	if [[ $OPENSHIFT ]]; then
		oc create sa pmm-server
		oc adm policy add-scc-to-user privileged -z pmm-server
		if [[ $OPERATOR_NS ]]; then
			timeout 30 oc delete clusterrolebinding $(kubectl get clusterrolebinding | grep 'pmm-psmdb-operator-' | awk '{print $1}') || :
			oc create clusterrolebinding pmm-psmdb-operator-cluster-wide --clusterrole=percona-server-mongodb-operator --serviceaccount=$namespace:pmm-server
			oc patch clusterrole/percona-server-mongodb-operator --type json -p='[{"op":"add","path": "/rules/-","value":{"apiGroups":["security.openshift.io"],"resources":["securitycontextconstraints"],"verbs":["use"],"resourceNames":["privileged"]}}]' -n $OPERATOR_NS
		else
			oc create rolebinding pmm-psmdb-operator-namespace-only --role percona-server-mongodb-operator --serviceaccount=$namespace:pmm-server
			oc patch role/percona-server-mongodb-operator --type json -p='[{"op":"add","path": "/rules/-","value":{"apiGroups":["security.openshift.io"],"resources":["securitycontextconstraints"],"verbs":["use"],"resourceNames":["privileged"]}}]'
		fi
		local additional_params="--set platform=openshift --set sa=pmm-server --set supresshttp2=false"
	fi

	retry 10 60 helm install monitoring --set imageTag=${IMAGE_PMM_SERVER#*:} --set imageRepo=${IMAGE_PMM_SERVER%:*} $additional_params https://percona-charts.storage.googleapis.com/pmm-server-$PMM_SERVER_VER.tgz
}

wait_certificate() {
	certificate="$1"

	for i in {1..10}; do
		kubectl wait --for=condition=Ready "certificate/$certificate" --timeout=60s
		sleep 1
	done
}

renew_certificate() {
	certificate="$1"

	wait_certificate "$certificate"
	desc "renew $certificate"

	local pod_name
	pod_name=$(kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}')

	local revision
	revision=$(kubectl_bin get certificate "$certificate" -o 'jsonpath={.status.revision}')

	kubectl_bin exec "$pod_name" -- /tmp/cmctl renew "$certificate"

	# wait for new revision
	for i in {1..10}; do
		local new_revision
		new_revision=$(kubectl_bin get certificate "$certificate" -o 'jsonpath={.status.revision}')
		if [ "$((revision + 1))" == "$new_revision" ]; then
			break
		fi
		sleep 1
	done
}

deploy_cmctl() {
	local service_account="cmctl"

	$sed -e "s/percona-server-mongodb-operator/$service_account/g" "${src_dir}/deploy/rbac.yaml" \
		| yq '(select(.rules).rules[] | select(contains({"apiGroups": ["cert-manager.io"]}))).resources += "certificates/status"' \
		| kubectl_bin apply -f -
	kubectl_bin apply -f "$conf_dir/cmctl.yml"
}

stop_cluster() {
	local cluster_name=$1
	local max_wait_time=${2:-120}

	local passed_time=0
	local sleep_time=1
	kubectl_bin patch psmdb "${cluster_name}" --type json -p='[{"op":"add","path":"/spec/pause","value":true}]'
	set +x
	echo -n 'Waiting for cluster stop'
	until [[ $(kubectl_bin get psmdb "${cluster_name}" -o jsonpath='{.status.mongos.ready}') -le 0 ]] \
		&& [[ $(kubectl_bin get deployment "${cluster_name}-mongos" -o jsonpath='{.status.replicas}') -le 0 ]] \
		&& [[ $(kubectl_bin get psmdb "${cluster_name}" -o jsonpath='{.status.replsets.cfg.ready}') -le 0 ]] \
		&& [[ $(kubectl_bin get psmdb "${cluster_name}" -o jsonpath='{.status.replsets.rs0.ready}') -le 0 ]]; do
		echo -n .
		passed_time=$((passed_time + sleep_time))
		sleep ${passed_time}
		if [[ ${passed_time} -gt ${max_wait_time} ]]; then
			echo "We've been waiting for cluster stop for too long. Exiting..."
			exit 1
		fi
	done
	echo
	set -x
}

start_cluster() {
	local cluster_name=$1

	kubectl_bin patch psmdb "${cluster_name}" --type json -p='[{"op":"add","path":"/spec/pause","value":false}]'
	wait_cluster_consistency "${cluster_name}"
}

format_pitr_target() {
	local target=$1

	echo ${target} | sed 's/T/ /' | sed 's/Z//'
}

get_latest_restorable_time() {
	local cluster=$1
	local first_timestamp
	local second_timestamp
	local retry=0

	# "pbm-agent status" can return different timestamp in first few seconds
	# we need to get it twice to be sure that timestamp was not changed
	until [[ $first_timestamp != "" && $first_timestamp != "null" && $first_timestamp == $second_timestamp ]]; do
		first_timestamp=$(kubectl_bin exec "$cluster-0" -c backup-agent -- pbm status -o json | jq '.backups.pitrChunks.pitrChunks | last | .range.end')
		sleep 5
		if [[ $first_timestamp != "" && $first_timestamp != "null" ]]; then
			second_timestamp=$(kubectl_bin exec "$cluster-0" -c backup-agent -- pbm status -o json | jq '.backups.pitrChunks.pitrChunks | last | .range.end')
		fi
		let retry+=1
		if [[ $retry -gt 30 ]]; then
			echo "Error: timeout waiting for timestamp"
			exit 1
		fi
	done

	$date -u -d @"$first_timestamp" "+%Y-%m-%dT%H:%M:%SZ"
}

get_latest_restorable_time_from_backup_object() {
	local backup_name=$1
	local latestRestorableTime
	local retry=0

	until [[ $latestRestorableTime != "" && $latestRestorableTime != "null" ]]; do
		sleep 5
		latestRestorableTime=$(kubectl_bin get psmdb-backup "$backup_name" -o jsonpath='{.status.latestRestorableTime}')
		let retry+=1
		if [[ $retry -gt 30 ]]; then
			echo "Error: timeout waiting for latestRestorableTime"
			exit 1
		fi
	done

	echo "$latestRestorableTime"
}

compare_latest_restorable_time() {
	local cluster=$1
	local backup_name=$2
	local latest_restorable_time
	local backup_time

	latest_restorable_time=$(get_latest_restorable_time "$cluster")
	backup_time=$(get_latest_restorable_time_from_backup_object "$backup_name")

	if [[ $latest_restorable_time != "$backup_time" ]]; then
		echo "Error: latestRestorableTime is not equal to the latest timestamp of the backup $backup_name: $latest_restorable_time != $backup_time"
		exit 1
	fi
}

pause_cluster() {
	local cluster_name=$1

	echo "Pausing cluster ${cluster_name}"

	kubectl_bin patch psmdb ${cluster_name} --type merge -p='{"spec": { "pause": true } }'
}

unpause_cluster() {
	local cluster_name=$1

	echo "Unpausing cluster ${cluster_name}"

	kubectl_bin patch psmdb ${cluster_name} --type merge -p='{"spec": { "pause": false } }'
}

disable_tls() {
	local cluster_name=$1

	echo "Disabling TLS for cluster ${cluster_name}"

	kubectl_bin patch psmdb ${cluster_name} --type merge -p='{"spec": { "unsafeFlags": { "tls": true }, "tls": { "mode": "disabled" } } }'
}

wait_for_cluster_state() {
	local cluster_name=$1
	local target_state=$2

	echo -n "Waiting for psmdb/${cluster_name} to reach ${target_state} state"
	local timeout=0
	until [[ $(kubectl_bin get psmdb ${cluster_name} -o jsonpath={.status.state}) =~ ${target_state} ]]; do
		sleep 1
		timeout=$((timeout + 1))
		echo -n '.'
		if [[ ${timeout} -gt 1500 ]]; then
			echo
			echo "Waiting timeout has been reached. Exiting..."
			exit 1
		fi
	done
	echo
	log "psmdb/${cluster_name} is ${target_state}: OK"
}

urlencode() {
	uri="$1"
	echo -n "$uri" | jq -s -R -r @uri
}

getUserData() {
	local secretName=$1
	local dataKey=$2
	urlencode "$(getSecretData "$secretName" "$dataKey")"
}

write_document() {
	local cmp_postfix="$1"
	local sleep_value=${2:-0}

	log 'write initial data, read from all'
	run_mongos \
		'use myApp\n db.test.insert({ x: 100500 })' \
		"myApp:myPass@$cluster-mongos.$namespace"
	sleep $sleep_value

	compare_mongos_cmd "find" "myApp:myPass@$cluster-mongos.$namespace" ${cmp_postfix}
}

write_initial_data() {
	desc 'create user myApp'
	run_mongos \
		'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' \
		"userAdmin:userAdmin123456@$cluster-mongos.$namespace"
	sleep 2
}

reset_collection() {
	desc 'reset data'

	run_mongos \
		'use myApp\n db.test.remove({})' \
		"myApp:myPass@$cluster-mongos.$namespace"
	sleep 2
	write_document '' '120'
}

get_latest_oplog_chunk_ts() {
	local cluster=$1
	echo $(kubectl_bin exec ${cluster}-rs0-0 -c backup-agent -- pbm status -o json | jq '.backups.pitrChunks.pitrChunks | last | .range.end')
}

format_date() {
	local timestamp=$1
	echo $(TZ=UTC $date -d@${timestamp} '+%Y-%m-%d %H:%M:%S')
}

get_bucket_name() {
	local backup_name=$1

	kubectl_bin get psmdb-backup $backup_name -o jsonpath='{.status.s3.bucket}'
}

check_recovery() {
	local backup_name=$1
	local restore_type=$2
	local restore_date=$3
	local cmp_postfix=$4
	local cluster_name=$5
	local backupSource=$6

	local latest_ts=$(get_latest_oplog_chunk_ts $cluster_name)

	desc "write more data before restore by $restore_type"
	run_mongos \
		'use myApp\n db.test.insert({ x: 100501 })' \
		"myApp:myPass@$cluster-mongos.$namespace"

	if [[ -n ${restore_date} ]]; then
		desc "Restoring to time $(format_date ${restore_date})"
		retries=0
		until [[ ${latest_ts} -gt ${restore_date} ]]; do
			if [[ $retries -gt 30 ]]; then
				echo "Last oplog chunk ($(format_date ${latest_ts})) is not greater than restore target ($(format_date ${restore_date}))"
				exit 1
			fi
			latest_ts=$(get_latest_oplog_chunk_ts $cluster_name)
			retries=$((retries + 1))
			echo "Waiting for last oplog chunk ($(format_date ${latest_ts})) to be greater than restore target ($(format_date ${restore_date}))"
			sleep 10
		done
	else
		desc "Restoring to latest"
		local current_ts=$(get_latest_oplog_chunk_ts $cluster_name)
		retries=0
		until [[ ${latest_ts} -gt ${current_ts} ]]; do
			if [[ $retries -gt 30 ]]; then
				echo "Timeout while waiting for last oplog chunk ($(format_date ${latest_ts}))"
				exit 1
			fi
			latest_ts=$(get_latest_oplog_chunk_ts $cluster_name)
			retries=$((retries + 1))
			echo "Waiting for last oplog chunk ($(format_date ${latest_ts})) to be 120 seconds older than starting chunk ($(format_date ${current_ts}))"
			sleep 10
		done
	fi

	if [ -z "$backupSource" ]; then
		desc "check restore by $restore_type"
		cat $test_dir/conf/restore.yml \
			| $sed -e "s/name:/name: restore-$backup_name/" \
			| $sed -e "s/backupName:/backupName: $backup_name/" \
			| $sed -e "/backupSource/,+8d" \
			| $sed -e "s/pitrType:/type: $restore_type/" \
			| if [ -z "$restore_date" ]; then $sed -e "/date:/d"; else $sed -e "s/date:/date: $(format_date ${restore_date})/"; fi \
			| kubectl_bin apply -f -
	else
		desc "check restore by $restore_type $backupSource"
		backup_dest=$(get_backup_dest "$backup_name")
		cat $test_dir/conf/restore.yml \
			| $sed -e "s/name:/name: restore-$backup_name/" \
			| $sed -e "/backupName/d" \
			| $sed -e "s/pitrType:/type: $restore_type/" \
			| if [ -z "$restore_date" ]; then $sed -e "/date:/d"; else $sed -e "s/date:/date: $(format_date ${restore_date})"/; fi \
			| $sed -e "s|DESTINATION|$backup_dest|" \
			| $sed -e "s|BUCKET-NAME|$(get_bucket_name "$backup_name")|" \
			| if [ -n "$selective_collection" ]; then yq eval '.spec.selective = {"namespaces": ["myApp.test"], "withUsersAndRoles": true}'; else yq; fi \
			| kubectl_bin apply -f -
	fi

	# fail faster if we don't reach requested status until some time
	wait_restore "$backup_name" "$cluster_name" "requested" "0" "1200"
	echo
	wait_restore "$backup_name" "$cluster_name" "ready" "0" "1600"
	echo
	set -o xtrace

	wait_for_running $cluster-rs0 3
	wait_for_running $cluster-cfg 3
	wait_for_running $cluster-mongos 3
	sleep 10

	compare_mongos_cmd "find" "myApp:myPass@$cluster-mongos.$namespace" "$cmp_postfix"
}

run_pitr_check() {
	local backup=$1
	local cluster=$2
	local find_prefix=$3

	wait_for_oplogs "${cluster}"
	local target_time=$(format_date $(get_latest_oplog_chunk_ts ${cluster}))

	log "dropping test collection"
	run_mongo 'use myApp\n db.test.drop()' "myApp:myPass@${cluster}-rs0.${namespace}"

	log "checking pitr... backup: ${backup} target: ${target_time}"
	cat $test_dir/conf/pitr.yml \
		| yq eval ".metadata.name = \"restore-${backup}\"" \
		| yq eval ".spec.backupName = \"${backup}\"" \
		| yq eval ".spec.pitr.date = \"${target_time}\"" \
		| kubectl_bin apply -f -

	wait_restore "${backup}" "${cluster}"
	compare_mongo_cmd "find" "myApp:myPass@${cluster}-rs0.${namespace}" "${find_prefix}" ".svc.cluster.local" "myApp" "test"
}

wait_for_oplogs() {
	local cluster1=$1

	local backup_last_write=$(kubectl_bin exec ${cluster}-rs0-0 -c backup-agent -- pbm status -o json | jq .backups.snapshot[0].restoreTo)

	local retries=0
	local last_chunk=$(get_latest_oplog_chunk_ts ${cluster})
	until [[ ${last_chunk} -gt ${backup_last_write} ]]; do
		if [[ $retries -gt 30 ]]; then
			log "Last oplog chunk ($(format_date ${last_chunk})) is not greater than last write ($(format_date ${backup_last_write}))"
			exit 1
		fi
		last_chunk=$(get_latest_oplog_chunk_ts ${cluster})
		retries=$((retries + 1))
		log "Waiting for last oplog chunk ($(format_date ${last_chunk})) to be greater than last write ($(format_date ${backup_last_write}))"
		sleep 10
	done
}

function check_backup_existence_aws() {
	bucket=$(echo "$1" | cut -d'/' -f1)
	key_prefix=$(echo "$1" | cut -d'/' -f2-)
	key=$2
	storage_name="aws-s3"
	retry=0

	until aws s3api head-object --bucket "$bucket" --key "${key_prefix}${key}" &>/dev/null; do
		if [ $retry -ge 30 ]; then
			echo "max retry count $retry reached. something went wrong with operator or kubernetes cluster"
			echo "Backup was not found in bucket -- $storage_name"
			exit 1
		fi
		echo "waiting for backup $bucket/$key_prefix/$key in $storage_name"
		sleep 10
		((retry += 1))
	done

	echo "Backup $bucket/$key_prefix/$key found in $storage_name"
}

function check_backup_existence_gcs() {
	backup_dest_gcp=$1
	obj=$2
	storage_name="gcp-cs"
	retry=0

	gcs_path="gs://${backup_dest_gcp}${obj}"

	until gsutil ls "$gcs_path" >/dev/null 2>&1; do
		if [ $retry -ge 30 ]; then
			echo "Max retry count $retry reached. Something went wrong with operator or Kubernetes cluster."
			echo "Backup was not found in bucket -- $storage_name"
			exit 1
		fi
		echo "Waiting for backup $gcs_path in $storage_name"
		sleep 10
		((retry += 1))
	done

	echo "Backup $gcs_path found in $storage_name"
}

function check_backup_existence_azure() {
	url_path=$(echo "$1" | sed 's|https://[^/]*\.blob\.core\.windows\.net/||')
	container=$(echo "$url_path" | cut -d'/' -f1)
	blob_prefix=$(echo "$url_path" | cut -d'/' -f2-)

	blob=$2
	storage_name="azure-blob"
	retry=0
	blob_path="${blob_prefix}${blob}"

	until az storage blob show --container-name "$container" --name "$blob_path" &>/dev/null; do
		if [ $retry -ge 30 ]; then
			echo "max retry count $retry reached. something went wrong with operator or kubernetes cluster"
			echo "Backup was not found in container -- $storage_name"
			exit 1
		fi
		echo "waiting for backup $blob_path in $storage_name"
		sleep 10
		((retry += 1))
	done

	echo "Backup $blob_path found in $storage_name"
}

check_backup_in_storage() {
	local backup=$1
	local storage_type=$2
	local replset=$3
	local file=${4:-"filelist.pbm"}

	local endpoint
	backup_dest=$(get_backup_dest "$backup" | $sed 's|https://engk8soperators.blob.core.windows.net/||')
	case ${storage_type} in
		s3)
			endpoint="s3.amazonaws.com"
			setup_aws_credentials
			check_backup_existence_aws "$backup_dest" "/${replset}/${file}"
			;;
		gcs)
			endpoint="storage.googleapis.com"
			setup_gcs_credentials
			check_backup_existence_gcs "$backup_dest" "/${replset}/${file}"
			;;
		azure)
			endpoint="engk8soperators.blob.core.windows.net"
			setup_azure_credentials
			check_backup_existence_azure "$backup_dest" "/${replset}/${file}"
			;;
		minio)
			endpoint="minio-service"
			until kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- \
				/usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 \
				/usr/bin/aws --endpoint-url http://${endpoint}:9000 s3 ls "s3://${backup_dest}/${replset}/${file}" \
				| grep "${file}"; do
				sleep 1
				let retry+=1
				if [ $retry -ge 60 ]; then
					log "Max retry count $retry reached. Something went wrong with writing backup"
					exit 1
				fi
			done
			;;
		*)
			echo "unsupported storage type: ${storage_type}"
			exit 1
			;;
	esac
}

function wait_all_pvc_resize() {
	local expected_size=$1
	local max_retry=${2:-120}
	local sleep_time=${3:-5}

	for pvc in $(kubectl_bin get pvc -l app.kubernetes.io/component=mongod -o name); do
		if ! wait_pvc_resize "$pvc" "$expected_size" "$max_retry" "$sleep_time"; then
			return 1
		fi
	done
	return 0
}

function wait_pvc_resize() {
	local pvc=$1
	local expected_size=$2
	local max_retry=${3:-120}
	local sleep_time=${4:-5}

	local retry=0
	echo "Waiting for $pvc to be resized"
	until [[ $(kubectl_bin get "$pvc" -o jsonpath='{.status.capacity.storage}') == "$expected_size" ]]; do
		if [[ $retry -ge $max_retry ]]; then
			echo
			echo "$pvc was not resized, max retries exceeded"
			return 1
		fi
		echo -n "."
		sleep "$sleep_time"

		retry=$((retry + 1))
	done
	echo
	echo "${pvc} was resized"
	return 0
}