sno-ready.sh

#!/bin/bash
#
# Helper script to validate if the SNO node contains all the necessary tunings
# usage: ./sno-ready.sh
# usage: ./sno-ready.sh <cluster-name>
# The script will validate the cluster configurations towards the latest cluster created by sno-install.sh if <cluster-name> is not present
# If cluster-name presents it will validate the cluster configurations towards the cluster with config file: instance/<cluster-name>/config-resolved.yaml
#

if ! type "yq" > /dev/null 2>&1; then
  echo "Cannot find yq in the path, please install yq on the node first. ref: https://github.com/mikefarah/yq#install"
  exit 1
fi

if ! type "jinja2" > /dev/null 2>&1; then
  echo "Cannot find jinja2 in the path, will install it with pip3 install jinja2-cli and pip3 install jinja2-cli[yaml]"
  pip3 install --user jinja2-cli
  pip3 install --user jinja2-cli[yaml]
fi

if ! type "oc" > /dev/null 2>&1; then
  echo "Cannot find oc in the path, please install oc on the node first. ref: https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest/"
  exit 1
fi

SSH="ssh -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -o LogLevel=quiet"

usage(){
	echo "Usage: $0 <cluster-name>"
	echo "If <cluster-name> is not present, it will validate the configurations on the newest cluster installed last time"
  echo "Example: $0"
  echo "Example: $0 sno130"
}

if [[ ( $@ == "--help") ||  $@ == "-h" ]]
then
  usage
  exit
fi

basedir="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"

# Color codes for better output (defined early for use in functions)
RED=$(tput setaf 1)
GREEN=$(tput setaf 2)
YELLOW=$(tput setaf 3)
BLUE=$(tput setaf 4)
MAGENTA=$(tput setaf 5)
CYAN=$(tput setaf 6)
WHITE=$(tput setaf 7)
BOLD=$(tput bold)
RESET=$(tput sgr0)

# Debug function (defined early as it's used during initialization)
debug(){
  if [[ "${DEBUG:-false}" == "true" ]]; then
    printf "${MAGENTA}[DEBUG]${RESET} %s\n" "$1"
  fi
}

cluster_name=$1

if [ -z "$cluster_name" ]; then
  cluster_name=$(ls -t "$basedir/instances" 2>/dev/null | head -1)
  debug "Auto-selected cluster: $cluster_name"
fi

cluster_workspace=$basedir/instances/$cluster_name
config_file=$cluster_workspace/config-resolved.yaml

day2_workspace=$cluster_workspace/day2
cluster_tunings_workspace=$day2_workspace/cluster-tunings
node_tunings_workspace=$day2_workspace/node-tunings
operators_workspace=$day2_workspace/operators
operators=$basedir/operators
templates=$basedir/templates

FILTER_FIELDS='
del(.metadata.creationTimestamp) |
del(.metadata.resourceVersion) |
del(.metadata.annotations) |
del(.metadata.uid) |
del(.metadata.generation) |
del(.status) |
del(.metadata.finalizers) |
del(.metadata.ownerReferences) |
del(.spec.clusterID) |
del(.metadata.managedFields)
'

OC='oc --kubeconfig='$cluster_workspace'/auth/kubeconfig'

ocp_release=$($OC version -o json 2>/dev/null | jq -r '.openshiftVersion')
ocp_y_version=$(echo "$ocp_release" | cut -d. -f 1-2)

ssh_priv_key_input=$(yq -r '.ssh_priv_key //""' "$config_file")
if [[ -n "${ssh_priv_key_input}" ]]; then
  ssh_key_path=$(eval echo "$ssh_priv_key_input")
  SSH+=" -i ${ssh_key_path}"
fi

# Enhanced output functions
info(){
  local msg1="$1"
  local msg2="$2"
  # Calculate display length accounting for multi-byte characters
  local len=${#msg1}
  local padding=$((80 - len))
  if [ $padding -lt 0 ]; then padding=1; fi
  printf "${GREEN}✓${RESET} %s%*s${GREEN}%s${RESET}\n" "$msg1" "$padding" "" "$msg2"
}
  
warn(){
  local msg1="$1"
  local msg2="$2"
  local len=${#msg1}
  local padding=$((80 - len))
  if [ $padding -lt 0 ]; then padding=1; fi
  printf "${YELLOW}⚠${RESET} %s%*s${YELLOW}%s${RESET}\n" "$msg1" "$padding" "" "$msg2"
}

error(){
  local msg1="$1"
  local msg2="$2"
  local len=${#msg1}
  local padding=$((80 - len))
  if [ $padding -lt 0 ]; then padding=1; fi
  printf "${RED}✗${RESET} %s%*s${RED}%s${RESET}\n" "$msg1" "$padding" "" "$msg2"
}

# Tree-structured output with proper alignment
tree_info(){
  local prefix="$1"
  local name="$2" 
  local status="$3"
  local msg1="$prefix $name"
  # Use byte length like other functions for consistency
  local len=${#msg1}
  local padding=$((80 - len))
  if [ $padding -lt 0 ]; then padding=1; fi
  printf "${GREEN}✓${RESET} %s%*s${GREEN}%s${RESET}\n" "$msg1" "$padding" "" "$status"
}

tree_warn(){
  local prefix="$1"
  local name="$2"
  local status="$3"
  local msg1="$prefix $name"
  # Use byte length like other functions for consistency
  local len=${#msg1}
  local padding=$((80 - len))
  if [ $padding -lt 0 ]; then padding=1; fi
  printf "${YELLOW}⚠${RESET} %s%*s${YELLOW}%s${RESET}\n" "$msg1" "$padding" "" "$status"
}

tree_error(){
  local prefix="$1"
  local name="$2"
  local status="$3"
  local msg1="$prefix $name"
  local len=${#msg1}
  local padding=$((80 - len))
  if [ $padding -lt 0 ]; then padding=1; fi
  printf "${RED}✗${RESET} %s%*s${RED}%s${RESET}\n" "$msg1" "$padding" "" "$status"
}

step(){
  printf "\n${BOLD}${BLUE}▶%s${RESET}\n" "$1"
}

header(){
  echo
  printf "${BOLD}${CYAN}%s${RESET}\n" "$1"
  printf "${CYAN}%s${RESET}\n" "$(printf '%.0s=' {1..60})"
}

separator(){
  printf "${CYAN}%s${RESET}\n" "$(printf '%.0s-' {1..60})"
}

short_path(){
  echo "$*" | sed -e "s;${basedir};\${basedir};g" -e "s;${HOME};\${HOME};g"
}

export_address(){
  export address=$($OC get node -o jsonpath='{..addresses[?(@.type=="InternalIP")].address}' | awk '{print $1;}')
}

# Display formatted diff with proper headers and limited output
show_config_diff(){
  local desired_file="$1"
  local live_file="$2" 
  local resource_name="$3"
  local max_lines="${4:-6}"
  local indent="${5:-      }"
  
  printf "%s${BOLD}${CYAN}Configuration Differences:${RESET}\n" "$indent"
  printf "%s${RED}[-] Desired Config${RESET}  ${GREEN}[+] Live Cluster${RESET}\n" "$indent"
  printf "%s${CYAN}%s${RESET}\n" "$indent" "$(printf '%.0s─' {1..60})"
  
  # Use a more compact unified diff format
  local diff_output
  diff_output=$(diff -u "$desired_file" "$live_file" 2>/dev/null | tail -n +3)
  
  if [ -n "$diff_output" ]; then
    local line_count=0
    echo "$diff_output" | while IFS= read -r line; do
      if [ $line_count -ge $max_lines ]; then
        break
      fi
      
      # Color code the diff lines
      case "${line:0:1}" in
        "+")
          printf "%s${GREEN}%s${RESET}\n" "$indent" "$line"
          ;;
        "-")
          printf "%s${RED}%s${RESET}\n" "$indent" "$line"
          ;;
        "@")
          printf "%s${BLUE}%s${RESET}\n" "$indent" "$line"
          ;;
        *)
          printf "%s%s\n" "$indent" "$line"
          ;;
      esac
      line_count=$((line_count + 1))
    done
    
    local total_lines
    total_lines=$(echo "$diff_output" | wc -l)
    if [ "$total_lines" -gt "$max_lines" ]; then
      printf "%s${MAGENTA}... (%d more lines)${RESET}\n" "$indent" $((total_lines - max_lines))
    fi
  else
    printf "%s${GREEN}✓ No differences found${RESET}\n" "$indent"
  fi
  
  echo
  printf "%s${BLUE}💡 Full diff: ${YELLOW}oc diff -f $resource_name${RESET}\n" "$indent"
}

# Validate environment and dependencies
validate_environment(){
  debug "Validating environment and dependencies"
  local validation_failed=false
  
  # Check required commands
  local required_commands=("yq" "jinja2" "oc")
  for cmd in "${required_commands[@]}"; do
    if ! command -v "$cmd" >/dev/null 2>&1; then
      error "Required command not found: $cmd" "Please install $cmd"
      validation_failed=true
    else
      debug "  ✓ Found command: $cmd"
    fi
  done
  
  # Check if KUBECONFIG will be valid
  if [[ -n "$KUBECONFIG" ]]; then
    debug "KUBECONFIG environment variable set to: $KUBECONFIG"
  fi
  
  if $validation_failed; then
    error "Environment validation failed" "Cannot continue"
    return 1
  fi
  
  debug "Environment validation passed"
  return 0
}

check_node(){
  step "Checking node status"
  if [ "$($OC get node -o jsonpath='{..conditions[?(@.type=="Ready")].status}')" = "True" ]; then
    info "Node" "ready"
  else
    warn "Node" "not ready"
  fi
}

check_pods(){
  step "Checking all pods"
  if [ "$($OC get pods -A | grep -vE "Running|Completed" | wc -l)" -gt 1 ]; then
    warn "Some pods are failing or creating."
    $OC get pods -A | grep -vE "Running|Completed"
  else
    info "No failing pods."
  fi
}

check_cluster_operators(){
  step "Checking cluster operators"
  for name in $($OC get co -o jsonpath={..metadata.name}); do
    local progressing=$($OC get co "$name" -o jsonpath='{..conditions[?(@.type=="Progressing")].status}')
    local available=$($OC get co "$name" -o jsonpath='{..conditions[?(@.type=="Available")].status}')
    local degraded=$($OC get co "$name" -o jsonpath='{..conditions[?(@.type=="Degraded")].status}')
    if [ "$available" = "True" ] && [ "$progressing" = "False" ] && [ "$degraded" = "False" ]; then
      info "cluster operator $name " "healthy"
    else
      warn "cluster operator $name " "unhealthy"
    fi
  done
}

check_workload_partitioning(){
  step "Checking workload partitioning"
  if [ "true" = "$(yq '.node_tunings.workload_partitioning.enabled' "$config_file")" ]; then
    info "workload_partition" "enabled in $config_file"
    if [ "$($OC get mc | grep 01-master-cpu-partitioning | wc -l)" -eq 1 ]; then
      info "mc 01-master-cpu-partitioning" "found"
    else
      warn "mc 01-master-cpu-partitioning" "not found"
    fi
  else
    warn "workload_partition" "not enabled in $config_file"
  fi
}

should_exclude_machine_config(){
  local file=$1
  local exclude_patterns=$(yq -r '.cluster_tunings.excludes[]' "$config_file" 2>/dev/null)

  if [[ -n "$exclude_patterns" && "$exclude_patterns" != "null" ]]; then
    for pattern in $exclude_patterns; do
      if [[ "$file" == *"$pattern"* ]]; then
        return 0
      fi
    done
  fi
  return 1
}

check_machine_config(){
  local file=$1
  local status=$2
  local mc_name
  mc_name=$(yq ".metadata.name" "$file")
  if [ "$status" = "included" ]; then
    if [ "$($OC get mc | grep "$mc_name" | wc -l)" -eq 1 ]; then
      tree_info " ├─" "$(basename "$file")" "included and created"
    else
      tree_error " ├─" "$(basename "$file")" "included but not created"
    fi
  fi

  if [ "$status" = "excluded" ]; then
    if [ "$($OC get mc | grep "$mc_name" | wc -l)" -eq 1 ]; then
      tree_error " ├─" "$(basename "$file")" "excluded but still found"
    else
      tree_info " ├─" "$(basename "$file")" "excluded and not found"
    fi
  fi
  
}

is_mc_file(){
  local file=$1
  if [[ $(yq ".kind" "$file") == "MachineConfig" ]]; then
    return 0
  else
    return 1
  fi
}

check_machine_configs(){
  step "Checking required machine configs"

  local cluster_tunings_version=$(yq '.cluster_tunings.version' "$config_file")
  if [[ -z "$cluster_tunings_version" || "$cluster_tunings_version" == "null" ]] || [[ "$cluster_tunings_version" == "none" ]]; then
    warn "Cluster tunings" "disabled in $config_file"
  else
    if [[ -d "$templates/day1/cluster-tunings/$cluster_tunings_version" ]]; then
      # Count total available files
      local tuning_files=$(ls "$templates/day1/cluster-tunings/$cluster_tunings_version"/*.yaml 2>/dev/null)
      local exclude_patterns=$(yq -r '.cluster_tunings.excludes[]' "$config_file" 2>/dev/null)
      debug "Exclude patterns: $exclude_patterns"
      for file in $tuning_files; do
        if is_mc_file "$file"; then
          if should_exclude_machine_config "$file"; then
            check_machine_config "$file" "excluded"
            debug "Excluded: $file"
          else
            check_machine_config "$file" "included"
            debug "Included: $file"
          fi
        fi
      done
    else
      warn "Cluster tunings" "version $cluster_tunings_version not found"
      return 1
    fi
  fi
}

check_machine_config_pools(){
  step "Checking machine config pool"
  local updated=$($OC get mcp master -o jsonpath='{..conditions[?(@.type=="Updated")].status}')
  local updating=$($OC get mcp master -o jsonpath='{..conditions[?(@.type=="Updating")].status}')
  local degraded=$($OC get mcp master -o jsonpath='{..conditions[?(@.type=="Degraded")].status}')
  if [ "$updated" = "True" ] && [ "$updating" = "False" ] && [ "$degraded" = "False" ]; then
    info "mcp master" "updated and not degraded"
  else
    warn "mcp master" "updating or degraded"
  fi
}

check_performance_profile(){
  step "Checking required performance profile"

  if [ "true" = "$(yq '.node_tunings.performance_profile.enabled' "$config_file")" ]; then
    info "Legend" "${RED}[-] Desired Config${RESET}  ${GREEN}[+] Live Cluster${RESET}"
    local desired_file="$node_tunings_workspace/performance-profile/performance-profile-final.yaml"
    if [ ! -f "$desired_file" ]; then
      error "Performance profile" "not found in $desired_file"
      return 0
    fi
    local pretty_desired_file="$node_tunings_workspace/performance-profile/performance-profile-pretty.yaml"
    local profile_name=$(yq ".metadata.name" "$desired_file")
    if [ "$($OC get performanceprofile | grep "$profile_name" | wc -l)" -eq 1 ]; then
      info "PerformanceProfile $profile_name found"
      local live_file="$node_tunings_workspace/performance-profile/performance-profile-live.yaml"
      yq eval "${FILTER_FIELDS}" "$desired_file" | yq '... comments=""' | yq -P 'sort_keys(..)' | yq eval --prettyPrint > "$pretty_desired_file"
      $OC get performanceprofile "$profile_name" -o yaml | yq '... comments=""' | yq -P 'sort_keys(..)' | yq eval "${FILTER_FIELDS}" | yq eval --prettyPrint > "$live_file"
      diff -q "$pretty_desired_file" "$live_file" > /dev/null
      if [ $? -eq 0 ]; then
        info "PerformanceProfile $profile_name is identical to the desired one."
      else
        warn "PerformanceProfile $profile_name" "configuration drift detected"
        show_config_diff "$pretty_desired_file" "$live_file" "$desired_file" 8 "    "
      fi
    else
      warn "PerformanceProfile $profile_name is not found."
    fi
  else
    warn "Performance profile" "disabled in $config_file"
  fi
}

check_tuned_profile(){
  step "Checking required tuned profile"

  if [ "true" = "$(yq '.node_tunings.tuned_profile.enabled' "$config_file")" ]; then
    info "Legend" "${RED}[-] Desired Config${RESET}  ${GREEN}[+] Live Cluster${RESET}"

    # Get all profiles from config
    local profiles
    readarray -t profiles < <(yq '.node_tunings.tuned_profile|keys[]|select(.!="enabled")' "$config_file" 2>/dev/null)
    if [[ ${#profiles[@]} -le 0 ]]; then
      warn "No tuned profiles configured" "skipping"
      return 0
    fi

    info "Configured tuned profiles: ${profiles[*]}"

    # Process each profile
    for profile in "${profiles[@]}"; do
      local desired_file="$node_tunings_workspace/tuned-profiles/tuned-$profile.yaml"
      local profile_name=$(yq ".metadata.name" "$desired_file")
      local pretty_desired_file="$node_tunings_workspace/tuned-profiles/tuned-$profile-pretty.yaml"
      local live_file="$node_tunings_workspace/tuned-profiles/tuned-$profile-live.yaml"
      yq eval "${FILTER_FIELDS}" "$desired_file" | yq '... comments=""' | yq -P 'sort_keys(..)' | yq eval --prettyPrint > "${pretty_desired_file}" 2>/dev/null;
      
      $OC get tuned "$profile_name" -n openshift-cluster-node-tuning-operator -o yaml | yq '... comments=""' | yq -P 'sort_keys(..)' | yq eval "${FILTER_FIELDS}" | yq eval --prettyPrint > "$live_file"
      diff -q "$pretty_desired_file" "$live_file" > /dev/null
      if [ $? -eq 0 ]; then
        info "TunedProfile $profile_name is identical to the desired one."
      else
        warn "TunedProfile $profile_name" "configuration drift detected"
        show_config_diff "$pretty_desired_file" "$live_file" "$desired_file" 8 "    "
      fi
    done
  else
    warn "Tuned profile" "disabled in $config_file"
  fi
}

diff_custom_resource(){
  local file=$1
  if [ ! -f "$file" ]; then
    debug "File not found: $file"
    return 1
  fi
  
  # Extract resource information with error handling
  local kind name ns
  kind=$(yq ".kind" "$file" 2>/dev/null)
  name=$(yq ".metadata.name" "$file" 2>/dev/null)
  ns=$(yq ".metadata.namespace // \"\"" "$file" 2>/dev/null)
  
  if [ -z "$kind" ] || [ -z "$name" ]; then
    tree_warn " ├─" "$(basename "$file")" "invalid YAML (missing kind or name)"
    return 1
  fi
  
  debug "Checking $kind/$name in namespace: ${ns:-default}"
  
  # Generate file names for comparison
  local desired_pretty_file live_pretty_file
  desired_pretty_file=$(echo "$file" | sed 's/\.yaml$/-pretty.yaml/')
  live_pretty_file=$(echo "$file" | sed 's/\.yaml$/-live-pretty.yaml/')

  # Create pretty version of desired file
  if ! yq eval "${FILTER_FIELDS}" "$file" | yq '... comments=""' | yq -P 'sort_keys(..)' | yq eval --prettyPrint > "$desired_pretty_file" 2>/dev/null; then
    tree_warn " ├─" "$(basename "$file")" "failed to process desired file"
    return 1
  fi

  # Get live resource and create pretty version
  local oc_cmd="$OC get $kind $name"
  if [ -n "$ns" ] && [ "$ns" != "null" ]; then
    oc_cmd="$oc_cmd -n $ns"
  fi
  
  if ! $oc_cmd -o yaml 2>/dev/null | yq '... comments=""' | yq -P 'sort_keys(..)' | yq eval "${FILTER_FIELDS}" | yq eval --prettyPrint > "$live_pretty_file" 2>/dev/null; then
    tree_warn " ├─" "$(basename "$file")" "resource not found in cluster"
    return 1
  fi

  # Compare files
  if diff -q "$desired_pretty_file" "$live_pretty_file" >/dev/null 2>&1; then
    tree_info " ├─" "$(basename "$file")" "identical to cluster"
  else
    tree_warn " ├─" "$(basename "$file")" "configuration drift detected"
    show_config_diff "$desired_pretty_file" "$live_pretty_file" "$file" 6 "      │     "
  fi
}

check_operator_day2(){
  local key=$1
  local operator_workspace="$day2_workspace/operators/$key"
  
  debug "Checking operator day2 configs for: $key"
  debug "Operator workspace: $operator_workspace"
  
  # Check if operator workspace exists
  if [ ! -d "$operator_workspace" ]; then
    debug "No day2 workspace found for operator: $key"
    return 0
  fi
  
  # Check base operator files
  if ls "$operator_workspace"/*.yaml >/dev/null 2>&1; then
    for file in "$operator_workspace"/*.yaml; do
      # Skip generated pretty files and live files
      if [ -f "$file" ] && [[ ! "$(basename "$file")" =~ -pretty\.yaml$ ]] && [[ ! "$(basename "$file")" =~ -live-pretty\.yaml$ ]]; then
        diff_custom_resource "$file"
      fi
    done
  fi

  # Check profile-specific files
  local profiles
  profiles=$(yq ".operators.$key.day2[].profile" "$config_file" 2>/dev/null)
  if [ -n "$profiles" ] && [ "$profiles" != "null" ]; then
    for profile in $profiles; do
      local profile_workspace="$operator_workspace/$profile"
      debug "Checking profile workspace: $profile_workspace"
      if [ -d "$profile_workspace" ] && ls "$profile_workspace"/*.yaml >/dev/null 2>&1; then
        for file in "$profile_workspace"/*.yaml; do
          # Skip generated pretty files and live files
          if [ -f "$file" ] && [[ ! "$(basename "$file")" =~ -pretty\.yaml$ ]] && [[ ! "$(basename "$file")" =~ -live-pretty\.yaml$ ]]; then
            diff_custom_resource "$file"
          fi
        done
      fi
    done
  else
    # Check default profile
    local default_workspace="$operator_workspace/default"
    if [ -d "$default_workspace" ] && ls "$default_workspace"/*.yaml >/dev/null 2>&1; then
      for file in "$default_workspace"/*.yaml; do
        # Skip generated pretty files and live files
        if [ -f "$file" ] && [[ ! "$(basename "$file")" =~ -pretty\.yaml$ ]] && [[ ! "$(basename "$file")" =~ -live-pretty\.yaml$ ]]; then
          diff_custom_resource "$file"
        fi
      done
    fi
  fi
  
}

check_operator(){
  local key=$1
  local operator_name=$(yq ".operators.$key.name" "$operators/operators.yaml")
  local operator_desc=$(yq ".operators.$key.desc" "$operators/operators.yaml")
  local ns
  
  # Check if has yaml file, then check the namespace
  if ls operators/"$key"/*.yaml >/dev/null 2>&1; then
    ns=$(yq '. | select(.kind == "Namespace")|.metadata.name' operators/"$key"/*.yaml)
  else
    ns=$(jinja2 operators/"$key"/*.yaml.j2 | yq ".metadata.namespace")
  fi

  debug "Operator: $operator_name, Namespace: $ns"

  local csv=$($OC get csv -n "$ns" | grep -E "$operator_name|$key" | wc -l)
  if [ "$csv" -eq 1 ]; then
    info "$operator_desc" "available"
    check_operator_day2 "$key"
    check_operator_readiness "$key"
  else
    warn "$operator_desc" "not available"
  fi

}

check_operator_readiness(){
  local key=$1
  #if the operator has readiness, then check the readiness one by one
  local readiness_checks
  readarray -t readiness_checks < <(yq ".operators.$key.readiness[].name" "$config_file" 2>/dev/null)
  if [[ ${#readiness_checks[@]} -gt 0 ]]; then
    local total=${#readiness_checks[@]}
    local idx=0
    for readiness_name in "${readiness_checks[@]}"; do
      idx=$((idx + 1))
      local variable=$(yq ".operators.$key.readiness[] | select(.name == \"$readiness_name\") | .variable" "$config_file" 2>/dev/null)
      local expected=$(yq ".operators.$key.readiness[] | select(.name == \"$readiness_name\") | .expected" "$config_file" 2>/dev/null)
      local status=$(eval "$variable")
      
      # Use └─ for last item, ├─ for others
      local prefix=" ├─"
      if [ $idx -eq $total ]; then
        prefix=" └─"
      fi
      
      if [ "$status" = "$expected" ]; then
        tree_info "$prefix" "$readiness_name" "passed"
      else
        tree_warn "$prefix" "$readiness_name" "failed"
      fi
    done
  fi
}

check_operators(){
  step "Checking operators"
  info "Legend" "${RED}[-] Desired Config${RESET}  ${GREEN}[+] Live Cluster${RESET}  ${BLUE}[@] Context${RESET}"
  
  local keys
  readarray -t keys < <(yq ".operators|keys" "$config_file" | yq '.[]')
  
  for ((k=0; k<${#keys[@]}; k++)); do
    local key="${keys[$k]}"
    if [ "true" = "$(yq ".operators.$key.enabled" "$config_file")" ]; then
      check_operator "$key"
    fi
  done
}

check_cluster_capabilities(){
  step "Checking cluster capabilities"
  
  local baseline_capability=$(yq -r ".cluster.capabilities.baselineCapabilitySet" "$config_file")
  debug "Baseline capability: ${baseline_capability}"

  if [ "$baseline_capability" = "None" ]; then
    info "Baseline capability" "None"

    local enabled_capabilities
    local config_capabilities
    readarray -t enabled_capabilities < <($OC get clusterversion version -o yaml | yq -r ".status.capabilities.enabledCapabilities[]")
    readarray -t config_capabilities < <(yq -r ".cluster.capabilities.additionalEnabledCapabilities[]" "$config_file")
    debug "Enabled capabilities: ${enabled_capabilities[*]}"
    debug "Config capabilities: ${config_capabilities[*]}"

    for capability in "${config_capabilities[@]}"; do
      if [ "$(printf '%s\n' "${enabled_capabilities[@]}" | grep -c "^${capability}$")" -eq 1 ]; then
        info "(Additional capability) $capability " "enabled"
      else
        warn "(Additional capability) $capability " "not enabled"
      fi
    done

    for capability in "${enabled_capabilities[@]}"; do
      if [ "$(printf '%s\n' "${config_capabilities[@]}" | grep -c "^${capability}$")" -eq 1 ]; then
        : # Expected capability, no action needed
      else
        warn "(Additional capability) $capability " "enabled but not needed"
      fi
    done
  else
    warn "Baseline capability" "${baseline_capability}"
  fi
}

check_catalog_sources(){
  step "Checking Catalog sources"

  if [ "true" = "$(yq '.catalog_sources.create_marketplace_ns' "$config_file")" ]; then
    if [ "$($OC get namespace openshift-marketplace 2>/dev/null | wc -l)" -eq 0 ]; then
      warn "Namespace openshift-marketplace" "configured but not found"
    else
      info "Namespace openshift-marketplace" "found"
    fi
  fi

  if [ "true" = "$(yq '.catalog_sources.update_operator_hub' "$config_file")" ]; then
    for source in $(yq -r '.catalog_sources.defaults[]' "$config_file"); do
      local disabled=$($OC get operatorhubs cluster -o yaml | yq '.spec.sources[] | select(.name == "'"$source"'") | .disabled // false')
      debug "OperatorHub $source disabled: $disabled"
      if [ "$disabled" = "true" ]; then
        warn "OperatorHub $source" "should not be disabled but disabled"
      else
        info "OperatorHub $source" "not disabled as needed"
      fi
    done
  fi
  
  if [ "true" = "$(yq '.catalog_sources.create_default_catalog_sources' "$config_file")" ]; then
    for source in $(yq -r '.catalog_sources.defaults[]' "$config_file"); do
      if [ "$($OC get catalogsource -n openshift-marketplace 2>/dev/null | grep "$source" | wc -l)" -eq 0 ]; then
        warn "Catalog $source" "not created"
      else
        info "Catalog $source" "created"
      fi
    done
  fi
}

check_installplans(){
  step "Checking InstallPlans"
  if [ "$($OC get installplans.operators.coreos.com -A 2>/dev/null | grep false | wc -l)" -gt 0 ]; then
    warn "InstallPlans below are not approved yet."
    $OC get installplans.operators.coreos.com -A | grep false
  else
    info "All InstallPlans have been approved or auto-approved."
  fi
}

check_extra_readiness(){
  local extra_readiness=$(yq '.readiness.extra_checks' "$config_file")
  if [ "$extra_readiness" = "null" ]; then
    : # No extra checks configured
  else
    local all_paths_config=$(yq '.readiness.extra_checks|join(" ")' "$config_file")
    local all_paths=$(eval echo "$all_paths_config")
    for d in $all_paths; do
      if [[ -d "$d" ]]; then
        step "Extra Checking $d"
        local check_files
        readarray -t check_files < <(find "${d}" -type f \( -name "*.yaml" -o -name "*.yaml.j2" -o -name "*.sh" \) | sort)
        for ((i=0; i<${#check_files[@]}; i++)); do
          local file="${check_files[$i]}"
          local output
          case "$file" in
            *.yaml)
              output=$($OC diff -f "$file" 2>&1)
              if [[ $? -ne 0 ]]; then
                warn "$(basename "$file")" "Failed"
              else
                info "$(basename "$file")" "Successful"
              fi
              echo "$output"
              ;;
            *.yaml.j2)
              output=$(jinja2 "$file" "$config_file" | $OC diff -f - 2>&1)
              if [[ $? -ne 0 ]]; then
                warn "$(basename "$file")" "Failed"
              else
                info "$(basename "$file")" "Successful"
              fi
              echo "$output"
              ;;
            *.sh)
              output=$(. "$file" "$config_file" "$cluster_workspace/auth/kubeconfig" 2>&1)
              if [[ $? -ne 0 ]]; then
                warn "$(basename "$file")" "Failed"
              else
                info "$(basename "$file")" "Successful"
              fi
              echo "$output"
              ;;
            *)
              warn "$file" "Skipped: unknown type"
              ;;
          esac
        done
      fi
    done
  fi
}

header "SNO Agent-Based Installer - Readiness Validation"

# Validate configuration file
if [ -f "$config_file" ]; then
  info "basedir" "$basedir"
  info "Configuration file" "$(short_path "$config_file")"
  info "Target cluster" "$cluster_name"
else
  error "Config file not found" "$(short_path "$config_file")"
  exit 1
fi

# Validate environment before proceeding
if ! validate_environment; then
  exit 1
fi

# Show debug status
if [[ "${DEBUG:-false}" == "true" ]]; then
  info "Debug mode" "ENABLED (set DEBUG=false to disable)"
else
  info "Debug mode" "disabled (set DEBUG=true to enable detailed logging)"
fi

debug "Script execution started at: $(date)"
debug "Configuration file: $config_file"
debug "Target cluster: $cluster_name"
debug "OpenShift version: ${ocp_release:-unknown}"

step "Gathering cluster information"
if ! $OC get clusterversion >/dev/null 2>&1; then
  error "Cluster is not reachable" "Check KUBECONFIG and connectivity"
  printf "${RED}KUBECONFIG: ${YELLOW}%s${RESET}\n" "${KUBECONFIG:-not set}"
  exit 1
fi
$OC get clusterversion

check_node
check_cluster_operators
export_address
check_pods
check_cluster_capabilities
check_catalog_sources
check_installplans
check_machine_config_pools
check_machine_configs
check_performance_profile
check_tuned_profile
check_operators
check_extra_readiness

header "SNO Readiness Check Complete - Summary"

debug "Script execution completed at: $(date)"

info "✅ All checks completed" "successfully"
info "📁 Kubeconfig location" "$(short_path "$cluster_workspace/auth/kubeconfig")"
info "⚙️ Configuration file" "$(short_path "$config_file")"
info "🎯 Target cluster" "$cluster_name"
info "🔧 OpenShift version" "$ocp_release"
separator
printf "${BOLD}${GREEN}🎉 SNO readiness validation completed!${RESET}\n"
if [[ "${DEBUG:-false}" != "true" ]]; then
  printf "${CYAN}For detailed debugging, run with: ${YELLOW}DEBUG=true %s${RESET}\n" "$0"
fi