Purge yq3 (#432)

Author: rarescosma

bin/common.bash

@@ -89,7 +89,7 @@ check_tools() {
   warn=0
   err=0
 
-  for executable in jq yq4 sops kubectl helm helmfile terraform; do
+  for executable in jq yq sops kubectl helm helmfile terraform; do
     if ! command -v "${executable}" >/dev/null; then
       log_error "Required dependency ${executable} missing"
       err=1
@@ -114,7 +114,7 @@ check_tools() {
     fi
   }
 
-  check_minor "$(yq4 '.[0].vars.terraform_version' "${req}")" "$(terraform --version --json | yq4 '.terraform_version')" "terraform"
+  check_minor "$(yq '.[0].vars.terraform_version' "${req}")" "$(terraform --version --json | yq '.terraform_version')" "terraform"
 
   if [[ "${warn}" != 0 ]]; then
     if [[ -t 1 ]]; then
@@ -133,13 +133,13 @@ validate_sops_config() {
     exit 1
   fi
 
-  rule_count=$(yq4 '.creation_rules | length' "${sops_config}")
+  rule_count=$(yq '.creation_rules | length' "${sops_config}")
   if [ "${rule_count:-0}" -gt 1 ]; then
     log_error "ERROR: SOPS config has more than one creation rule."
     exit 1
   fi
 
-  fingerprints=$(yq4 '.creation_rules[0].pgp' "${sops_config}")
+  fingerprints=$(yq '.creation_rules[0].pgp' "${sops_config}")
   if ! [[ "${fingerprints}" =~ ^[A-Z0-9,' ']+$ ]]; then
     log_error "ERROR: SOPS config contains no or invalid PGP keys."
     log_error "fingerprints=${fingerprints}"
@@ -176,7 +176,7 @@ append_trap() {
 
 # Write PGP fingerprints to SOPS config
 sops_config_write_fingerprints() {
-  yq4 -n '.creation_rules[0].pgp = "'"${1}"'"' >"${sops_config}" ||
+  yq -n '.creation_rules[0].pgp = "'"${1}"'"' >"${sops_config}" ||
     (log_error "Failed to write fingerprints" && rm "${sops_config}" && exit 1)
 }
 
@@ -331,7 +331,7 @@ ck8s_kubespray_version_check() {
   pushd "${root_path}" || exit
   version=$(git describe --exact-match --tags 2>/dev/null || git rev-parse HEAD)
   popd || exit
-  version_in_config=$(yq4 .ck8sKubesprayVersion "${config_path}/group_vars/all/ck8s-kubespray-general.yaml")
+  version_in_config=$(yq .ck8sKubesprayVersion "${config_path}/group_vars/all/ck8s-kubespray-general.yaml")
 
   if [[ -z "${version_in_config}" || "${version_in_config}" == "null" ]]; then
     log_error "ERROR: No version set. Ensure that ${config_path}/group_vars/all/ck8s-kubespray-general.yaml exists and has ck8sKubesprayVersion set."
@@ -384,10 +384,10 @@ ops_kubectl() { # <prefix> <args...>
 assign_host() {
   local node=$1
   # Check for control plane nodes
-  control_plane_label=$(yq4 .control_plane_label "${config_path}/group_vars/all/ck8s-kubespray-general.yaml")
+  control_plane_label=$(yq .control_plane_label "${config_path}/group_vars/all/ck8s-kubespray-general.yaml")
   # Check for AMS
-  primary_group_label=$(yq4 .group_label_primary "${config_path}/group_vars/all/ck8s-kubespray-general.yaml")
-  secondary_group_label=$(yq4 .group_label_secondary "${config_path}/group_vars/all/ck8s-kubespray-general.yaml")
+  primary_group_label=$(yq .group_label_primary "${config_path}/group_vars/all/ck8s-kubespray-general.yaml")
+  secondary_group_label=$(yq .group_label_secondary "${config_path}/group_vars/all/ck8s-kubespray-general.yaml")
   if [[ $(ops_kubectl "$prefix" get node "$node" -ojson | jq ".metadata.labels | has(\"${control_plane_label}\")") == "true" ]]; then
     target_group="kube_control_plane"
     if [[ "$(group_exists "${config[groups_inventory_file]}" "$target_group")" != "true" ]]; then

docs/citycloud-snippets.md

@@ -85,7 +85,7 @@ done
 
 If all went well the infrastructure should now be up and running
 
-## Compliantkubernetes kubespray
+## Welkin Kubespray
 
 To set up kubernetes with compliantkubernetes-kubespray you can follow these steps.
 

docs/migrate-from-cluster.md

@@ -47,7 +47,7 @@ This has only been tested on CityCloud and might not look the same for other clo
     export TF_VAR_ssh_pub_key_sc="${CK8S_CONFIG_PATH}/ssh/id_rsa_sc.pub"
     export TF_VAR_ssh_pub_key_wc="${CK8S_CONFIG_PATH}/ssh/id_rsa_wc.pub"
     export TF_DATA_DIR="${CK8S_CONFIG_PATH}/.state/.terraform"
-    export TF_WORKSPACE=$(yq r ${CK8S_CONFIG_PATH}/config.yaml environment_name)
+    export TF_WORKSPACE=$(yq ".environment_name" ${CK8S_CONFIG_PATH}/config.yaml)
 
     terraform state show module.service_cluster.module.network.openstack_networking_network_v2.network
     .

release/qa-checklist.md

@@ -53,11 +53,11 @@ Elastisys Welkin® Kubespray
 
     ```bash
     # configure
-    yq4 -i '.grafana.user.oidc.allowedDomains += ["example.com"]' "${CK8S_CONFIG_PATH}/sc-config.yaml"
-    yq4 -i '.grafana.ops.oidc.allowedDomains += ["example.com"]' "${CK8S_CONFIG_PATH}/sc-config.yaml"
-    yq4 -i 'with(.opensearch.extraRoleMappings[]; with(select(.mapping_name != "all_access"); .definition.users += ["[email protected]"]))' "${CK8S_CONFIG_PATH}/sc-config.yaml"
-    yq4 -i '.user.adminUsers += ["[email protected]"]' "${CK8S_CONFIG_PATH}/wc-config.yaml"
-    yq4 -i '.dex.enableStaticLogin = true' "${CK8S_CONFIG_PATH}/sc-config.yaml"
+    yq -i '.grafana.user.oidc.allowedDomains += ["example.com"]' "${CK8S_CONFIG_PATH}/sc-config.yaml"
+    yq -i '.grafana.ops.oidc.allowedDomains += ["example.com"]' "${CK8S_CONFIG_PATH}/sc-config.yaml"
+    yq -i 'with(.opensearch.extraRoleMappings[]; with(select(.mapping_name != "all_access"); .definition.users += ["[email protected]"]))' "${CK8S_CONFIG_PATH}/sc-config.yaml"
+    yq -i '.user.adminUsers += ["[email protected]"]' "${CK8S_CONFIG_PATH}/wc-config.yaml"
+    yq -i '.dex.enableStaticLogin = true' "${CK8S_CONFIG_PATH}/sc-config.yaml"
 
     # apply from the apps repository
     ./bin/ck8s apply sc
@@ -69,8 +69,8 @@ Elastisys Welkin® Kubespray
     <details><summary>Commands</summary>
 
     ```sh
-    yq4 -i '.grafana.user.trailingDots = false' "${CK8S_CONFIG_PATH}/sc-config.yaml"
-    yq4 -i '.grafana.ops.trailingDots = false' "${CK8S_CONFIG_PATH}/sc-config.yaml"
+    yq -i '.grafana.user.trailingDots = false' "${CK8S_CONFIG_PATH}/sc-config.yaml"
+    yq -i '.grafana.ops.trailingDots = false' "${CK8S_CONFIG_PATH}/sc-config.yaml"
 
     # apply from the apps repository
     ./bin/ck8s ops helmfile sc -lapp=grafana diff
@@ -89,7 +89,7 @@ Elastisys Welkin® Kubespray
 - [ ] Successful `./bin/ck8s test sc|wc` from the apps repository
 - [ ] If possible let the environment stabilise into a steady state after the install
     - Best is to perform the install at the end of the day to give it the night to stabilise.
-    - Otherwise give it at least one to two hours to stabilise if possible.
+    - Otherwise, give it at least one to two hours to stabilise if possible.
 
 #### Automated tests
 
@@ -477,11 +477,11 @@ Elastisys Welkin® Kubespray
 
     ```bash
     # configure
-    yq4 -i '.grafana.user.oidc.allowedDomains += ["example.com"]' "${CK8S_CONFIG_PATH}/sc-config.yaml"
-    yq4 -i '.grafana.ops.oidc.allowedDomains += ["example.com"]' "${CK8S_CONFIG_PATH}/sc-config.yaml"
-    yq4 -i 'with(.opensearch.extraRoleMappings[]; with(select(.mapping_name != "all_access"); .definition.users += ["[email protected]"]))' "${CK8S_CONFIG_PATH}/sc-config.yaml"
-    yq4 -i '.user.adminUsers += ["[email protected]"]' "${CK8S_CONFIG_PATH}/wc-config.yaml"
-    yq4 -i '.dex.enableStaticLogin = true' "${CK8S_CONFIG_PATH}/sc-config.yaml"
+    yq -i '.grafana.user.oidc.allowedDomains += ["example.com"]' "${CK8S_CONFIG_PATH}/sc-config.yaml"
+    yq -i '.grafana.ops.oidc.allowedDomains += ["example.com"]' "${CK8S_CONFIG_PATH}/sc-config.yaml"
+    yq -i 'with(.opensearch.extraRoleMappings[]; with(select(.mapping_name != "all_access"); .definition.users += ["[email protected]"]))' "${CK8S_CONFIG_PATH}/sc-config.yaml"
+    yq -i '.user.adminUsers += ["[email protected]"]' "${CK8S_CONFIG_PATH}/wc-config.yaml"
+    yq -i '.dex.enableStaticLogin = true' "${CK8S_CONFIG_PATH}/sc-config.yaml"
 
     # apply from the apps repository
     ./bin/ck8s apply sc
@@ -493,8 +493,8 @@ Elastisys Welkin® Kubespray
     <details><summary>Commands</summary>
 
     ```sh
-    yq4 -i '.grafana.user.trailingDots = false' "${CK8S_CONFIG_PATH}/sc-config.yaml"
-    yq4 -i '.grafana.ops.trailingDots = false' "${CK8S_CONFIG_PATH}/sc-config.yaml"
+    yq -i '.grafana.user.trailingDots = false' "${CK8S_CONFIG_PATH}/sc-config.yaml"
+    yq -i '.grafana.ops.trailingDots = false' "${CK8S_CONFIG_PATH}/sc-config.yaml"
 
     # apply from the apps repository
     ./bin/ck8s ops helmfile sc -lapp=grafana diff

rook/scripts/autoconfig.sh

@@ -7,7 +7,7 @@ addressInSubnet() {
 
 # fn <key> <addresses>
 updatePeers() {
-  peers="$(yq4 ".commons * .clusters.${cluster} | .networkPolicies.${1} | .[].cidr" "${CK8S_CONFIG_PATH}/rook/values.yaml")"
+  peers="$(yq ".commons * .clusters.${cluster} | .networkPolicies.${1} | .[].cidr" "${CK8S_CONFIG_PATH}/rook/values.yaml")"
 
   local -a update
 
@@ -22,14 +22,14 @@ updatePeers() {
     update+=("${address}/32")
   done
 
-  old="$(yq4 -oj -I0 '[split(" ") | sort | .[] | {"cidr": .}]' <<<"$peers")"
-  new="$(yq4 -oj -I0 '[split(" ") | sort | unique | .[] | {"cidr": .}]' <<<"${update[*]}")"
+  old="$(yq -oj -I0 '[split(" ") | sort | .[] | {"cidr": .}]' <<<"$peers")"
+  new="$(yq -oj -I0 '[split(" ") | sort | unique | .[] | {"cidr": .}]' <<<"${update[*]}")"
 
-  if ! diff -u3 --color --label "current .commons * .clusters.${cluster} | .${1}" <(yq4 -P <<<"$old") --label "update .cluster.${cluster}.${1}" <(yq4 -P <<<"$new"); then
+  if ! diff -u3 --color --label "current .commons * .clusters.${cluster} | .${1}" <(yq -P <<<"$old") --label "update .cluster.${cluster}.${1}" <(yq -P <<<"$new"); then
     echo -n "apply update? [Y/n]: "
     read -r reply
     if [[ "${reply}" =~ ^(Y|y|)$ ]]; then
-      yq4 -i ".clusters.${cluster}.networkPolicies.${1} = ${new}" "${CK8S_CONFIG_PATH}/rook/values.yaml"
+      yq -i ".clusters.${cluster}.networkPolicies.${1} = ${new}" "${CK8S_CONFIG_PATH}/rook/values.yaml"
     fi
   else
     echo "- ${cluster}/networkPolicies/${1}: up to date"
@@ -42,7 +42,7 @@ if [[ -z "${CK8S_CONFIG_PATH:-}" ]]; then
 elif [[ -z "${1:-}" ]]; then
   echo "err: missing cluster name" >&2
   exit
-elif [[ "$(yq4 ".clusters | keys | [\"${1}\"] - . | length" "${CK8S_CONFIG_PATH}/rook/values.yaml")" == "1" ]]; then
+elif [[ "$(yq ".clusters | keys | [\"${1}\"] - . | length" "${CK8S_CONFIG_PATH}/rook/values.yaml")" == "1" ]]; then
   echo "err: invalid cluster name" >&2
   exit
 fi
@@ -56,15 +56,15 @@ else
 fi
 
 apiserverAddresses="$(
-  kubectl get no -lnode-role.kubernetes.io/control-plane= -oyaml | yq4 '[
+  kubectl get no -lnode-role.kubernetes.io/control-plane= -oyaml | yq '[
   .items[] | [
     .status.addresses[] | select(.type == "InternalIP") | .address
   ] + (
     .metadata.annotations | [."projectcalico.org/IPv4IPIPTunnelAddr", ."projectcalico.org/IPv4WireguardInterfaceAddr"]) | .[]] | sort | .[]'
 )"
 
 nodeAddresses="$(
-  kubectl get no -oyaml | yq4 '[
+  kubectl get no -oyaml | yq '[
   .items[] | [
     .status.addresses[] | select(.type == "InternalIP") | .address
   ] + (
@@ -97,12 +97,12 @@ pspcrds=(
   k8spspvolumetypes.constraints.gatekeeper.sh
 )
 
-if [[ "$(yq4 ".commons * .clusters.${cluster} | .podSecurityPolicies.enabled" "${CK8S_CONFIG_PATH}/rook/values.yaml")" != "true" ]]; then
+if [[ "$(yq ".commons * .clusters.${cluster} | .podSecurityPolicies.enabled" "${CK8S_CONFIG_PATH}/rook/values.yaml")" != "true" ]]; then
   if kubectl get crds "${pspcrds[@]}" &>/dev/null; then
     echo -n "- enable Gatekeeper podsecuritypolicies? [Y/n]: "
     read -r reply
     if [[ "${reply}" =~ ^(Y|y|)$ ]]; then
-      yq4 -i ".clusters.${cluster}.podSecurityPolicies.enabled = true" "${CK8S_CONFIG_PATH}/rook/values.yaml"
+      yq -i ".clusters.${cluster}.podSecurityPolicies.enabled = true" "${CK8S_CONFIG_PATH}/rook/values.yaml"
     fi
   else
     echo "- warning: Gatekeeper constraints not available"
@@ -112,13 +112,13 @@ else
 fi
 
 echo "checking service monitors..."
-if [[ "$(yq4 ".commons * .clusters.${cluster} | .monitoring.installServiceMonitors" "${CK8S_CONFIG_PATH}/rook/values.yaml")" != "true" ]]; then
+if [[ "$(yq ".commons * .clusters.${cluster} | .monitoring.installServiceMonitors" "${CK8S_CONFIG_PATH}/rook/values.yaml")" != "true" ]]; then
   if kubectl get crd prometheuses.monitoring.coreos.com &>/dev/null; then
     if [[ -n "$(kubectl get po -A -l app.kubernetes.io/name=prometheus 2>/dev/null)" ]]; then
       echo -n "- install Prometheus service monitors? [Y/n]: "
       read -r reply
       if [[ "${reply}" =~ ^(Y|y|)$ ]]; then
-        yq4 -i ".clusters.${cluster}.monitoring.installServiceMonitors = true" "${CK8S_CONFIG_PATH}/rook/values.yaml"
+        yq -i ".clusters.${cluster}.monitoring.installServiceMonitors = true" "${CK8S_CONFIG_PATH}/rook/values.yaml"
       fi
     else
       echo "- note: Prometheus not available"
@@ -131,12 +131,12 @@ else
 fi
 
 echo "checking dashboards..."
-if [[ "$(yq4 ".commons * .clusters.${cluster} | .monitoring.installGrafanaDashboards" "${CK8S_CONFIG_PATH}/rook/values.yaml")" != "true" ]]; then
+if [[ "$(yq ".commons * .clusters.${cluster} | .monitoring.installGrafanaDashboards" "${CK8S_CONFIG_PATH}/rook/values.yaml")" != "true" ]]; then
   if [[ -n "$(kubectl get po -A -l app.kubernetes.io/name=grafana 2>/dev/null)" ]]; then
     echo -n "- install Grafana dashboards? [Y/n]: "
     read -r reply
     if [[ "${reply}" =~ ^(Y|y|)$ ]]; then
-      yq4 -i ".clusters.${cluster}.monitoring.installGrafanaDashboards = true" "${CK8S_CONFIG_PATH}/rook/values.yaml"
+      yq -i ".clusters.${cluster}.monitoring.installGrafanaDashboards = true" "${CK8S_CONFIG_PATH}/rook/values.yaml"
     fi
   else
     echo "- note: Grafana not available"
@@ -146,13 +146,13 @@ else
 fi
 
 echo "checking rules..."
-if [[ "$(yq4 ".commons * .clusters.${cluster} | .monitoring.installPrometheusRules" "${CK8S_CONFIG_PATH}/rook/values.yaml")" != "true" ]]; then
+if [[ "$(yq ".commons * .clusters.${cluster} | .monitoring.installPrometheusRules" "${CK8S_CONFIG_PATH}/rook/values.yaml")" != "true" ]]; then
   if kubectl get crd prometheusrules.monitoring.coreos.com &>/dev/null; then
     if [[ -n "$(kubectl get po -A -l app.kubernetes.io/name=thanos 2>/dev/null)" ]]; then
       echo -n "- install Prometheus rules? [Y/n]: "
       read -r reply
       if [[ "${reply}" =~ ^(Y|y|)$ ]]; then
-        yq4 -i ".clusters.${cluster}.monitoring.installPrometheusRules = true" "${CK8S_CONFIG_PATH}/rook/values.yaml"
+        yq -i ".clusters.${cluster}.monitoring.installPrometheusRules = true" "${CK8S_CONFIG_PATH}/rook/values.yaml"
       fi
     else
       echo "- note: Thanos not available"

rook/scripts/test-rook.sh

@@ -63,11 +63,11 @@ function testOSDs {
 #   2. cluster
 function testMons {
   numberOfMons=$(kubectl -n rook-ceph get deployments.apps -o=name | grep -c rook-ceph-mon)
-  numberOfMonsDesired=$(yq4 ".commons.mon.count" "$CONFIG_FILE")
+  numberOfMonsDesired=$(yq ".commons.mon.count" "$CONFIG_FILE")
   monDeployments=()
   read -r -d '' -a monDeployments <<<"$(kubectl -n rook-ceph get deployments.apps -o=json | jq -r '.items[].metadata.name' | grep "rook-ceph-mon")"
   if [[ -z $numberOfMonsDesired || $numberOfMonsDesired = "null" ]]; then
-    numberOfMonsDesired=$(yq4 ".cephClusterSpec.mon.count" "${here}/../helmfile.d/upstream/rook-ceph-cluster/values.yaml")
+    numberOfMonsDesired=$(yq ".cephClusterSpec.mon.count" "${here}/../helmfile.d/upstream/rook-ceph-cluster/values.yaml")
   fi
   if [[ ! $numberOfMons = "$numberOfMonsDesired" ]]; then
     echo "${numberOfMons}/${numberOfMonsDesired} deployments exist ❌"
@@ -87,20 +87,20 @@ function testMons {
 #   2. cluster
 function testMgrs {
   numberOfMgrs=$(kubectl -n rook-ceph get deployments.apps -o=name | grep -c rook-ceph-mgr)
-  numberOfMgrsDesired=$(yq4 ".commons.mgr.count" "$CONFIG_FILE")
+  numberOfMgrsDesired=$(yq ".commons.mgr.count" "$CONFIG_FILE")
   mgrDeployments=()
   read -r -d '' -a mgrDeployments <<<"$(kubectl -n rook-ceph get deployments.apps -o=json | jq -r '.items[].metadata.name' | grep "rook-ceph-mgr")"
   if [[ -z "$numberOfMgrsDesired" || "$numberOfMgrsDesired" = "null" ]]; then
     if [[ "${cluster}" = "sc" ]]; then
       TMP_DIR=$(mktemp -p /tmp -d rook-ceph-test.XXXXXX)
       append_trap "rm -rf ${TMP_DIR}" EXIT
       helmfile -e service write-values --output-file-template "${TMP_DIR}/{{ .State.BaseName }}-{{ .State.AbsPathSHA1 }}/{{ .Release.Name}}.yaml" &>/dev/null
-      numberOfMgrsDesired=$(yq4 ".cephClusterSpec.mgr.count" "$(find "${TMP_DIR}"/helmfile-*/ -name "rook-ceph-cluster.yaml")")
+      numberOfMgrsDesired=$(yq ".cephClusterSpec.mgr.count" "$(find "${TMP_DIR}"/helmfile-*/ -name "rook-ceph-cluster.yaml")")
     elif [[ "${cluster}" = "wc" ]]; then
       TMP_DIR=$(mktemp -p /tmp -d rook-ceph-test.XXXXXX)
       append_trap "rm -rf ${TMP_DIR}" EXIT
       helmfile -e workload write-values --output-file-template "${TMP_DIR}/{{ .State.BaseName }}-{{ .State.AbsPathSHA1 }}/{{ .Release.Name}}.yaml" &>/dev/null
-      numberOfMgrsDesired=$(yq4 ".cephClusterSpec.mgr.count" "$(find "${TMP_DIR}"/helmfile-*/ -name "rook-ceph-cluster.yaml")")
+      numberOfMgrsDesired=$(yq ".cephClusterSpec.mgr.count" "$(find "${TMP_DIR}"/helmfile-*/ -name "rook-ceph-cluster.yaml")")
     fi
   fi
   if [[ ! $numberOfMgrs = "$numberOfMgrsDesired" ]]; then
@@ -232,9 +232,9 @@ function test_rook() {
     targets=(
       "serviceMonitor/rook-ceph/rook-ceph-mgr/0 1"
     )
-    if [[ $cluster = "sc" && $(yq4 ".clusters.service.monitoring.installServiceMonitors" "$CONFIG_FILE") = true ]]; then
+    if [[ $cluster = "sc" && $(yq ".clusters.service.monitoring.installServiceMonitors" "$CONFIG_FILE") = true ]]; then
       test_targets_retry "svc/kube-prometheus-stack-prometheus" "${targets[@]}"
-    elif [[ $cluster = "wc" && $(yq4 ".clusters.workload.monitoring.installServiceMonitors" "$CONFIG_FILE") = true ]]; then
+    elif [[ $cluster = "wc" && $(yq ".clusters.workload.monitoring.installServiceMonitors" "$CONFIG_FILE") = true ]]; then
       test_targets_retry "svc/kube-prometheus-stack-prometheus" "${targets[@]}"
     else
       echo "ServiceMontiors not enabled in $cluster - Skipping"

scripts/migration/lib.sh

@@ -71,7 +71,7 @@ config_version() {
   local prefix="${1}"
 
   local version
-  version="$(yq4 ".ck8sKubesprayVersion" <<<"${CONFIG["${prefix}"]}")"
+  version="$(yq ".ck8sKubesprayVersion" <<<"${CONFIG["${prefix}"]}")"
 
   VERSION["${prefix}-config"]="${version}"
   version="${version#v}"