Merge pull request #6541 from XiShanYongYe-Chang/kube-cache-mutation-detector

enable mutation detection in e2e

Author: karmada-bot

.github/workflows/ci.yml

@@ -144,6 +144,7 @@ jobs:
       - name: setup e2e test environment
         run: |
           export CLUSTER_VERSION=kindest/node:${{ matrix.k8s }}
+          export KUBE_CACHE_MUTATION_DETECTOR=true
           hack/local-up-karmada.sh
       - name: run e2e
         run: |

artifacts/deploy/karmada-controller-manager.yaml

@@ -31,6 +31,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
+            - name: KUBE_CACHE_MUTATION_DETECTOR
+              value: "{{KUBE_CACHE_MUTATION_DETECTOR}}"
           command:
             - /bin/karmada-controller-manager
             - --kubeconfig=/etc/karmada/config/karmada.config

artifacts/deploy/karmada-descheduler.yaml

@@ -31,6 +31,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
+            - name: KUBE_CACHE_MUTATION_DETECTOR
+              value: "{{KUBE_CACHE_MUTATION_DETECTOR}}"
           command:
             - /bin/karmada-descheduler
             - --kubeconfig=/etc/karmada/config/karmada.config

artifacts/deploy/karmada-metrics-adapter.yaml

@@ -32,6 +32,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
+            - name: KUBE_CACHE_MUTATION_DETECTOR
+              value: "{{KUBE_CACHE_MUTATION_DETECTOR}}"
           command:
             - /bin/karmada-metrics-adapter
             - --kubeconfig=/etc/karmada/config/karmada.config

artifacts/deploy/karmada-scheduler-estimator.yaml

@@ -31,6 +31,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
+            - name: KUBE_CACHE_MUTATION_DETECTOR
+              value: "{{KUBE_CACHE_MUTATION_DETECTOR}}"
           command:
             - /bin/karmada-scheduler-estimator
             - --kubeconfig=/etc/{{member_cluster_name}}-kubeconfig

artifacts/deploy/karmada-scheduler.yaml

@@ -44,6 +44,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
+            - name: KUBE_CACHE_MUTATION_DETECTOR
+              value: "{{KUBE_CACHE_MUTATION_DETECTOR}}"
           command:
             - /bin/karmada-scheduler
             - --kubeconfig=/etc/karmada/config/karmada.config

artifacts/deploy/karmada-search.yaml

@@ -32,6 +32,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
+            - name: KUBE_CACHE_MUTATION_DETECTOR
+              value: "{{KUBE_CACHE_MUTATION_DETECTOR}}"
           command:
             - /bin/karmada-search
             - --kubeconfig=/etc/karmada/config/karmada.config

hack/deploy-karmada.sh

@@ -19,6 +19,9 @@ set -o nounset
 # This script deploy karmada control plane to any cluster you want.	REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 # This script depends on utils in: ${REPO_ROOT}/hack/util.sh
 
+# Do not run the mutation detector by default on the local karmada instance.
+KUBE_CACHE_MUTATION_DETECTOR="${KUBE_CACHE_MUTATION_DETECTOR:-false}"
+
 REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 CERT_DIR=${CERT_DIR:-"${HOME}/.karmada"}
 mkdir -p "${CERT_DIR}" &>/dev/null ||  mkdir -p "${CERT_DIR}"
@@ -185,6 +188,19 @@ function installCRDs() {
     kubectl --context="${context_name}" apply -k "${crd_path}"/_crds
 }
 
+function deploy_karmada_component() {
+  local component_name=$1
+  local temp_dir=$2
+  local wait_label=$3
+  
+  local component_yaml_tmp="${temp_dir}/${component_name}.yaml"
+  cp "${REPO_ROOT}/artifacts/deploy/${component_name}.yaml" "${component_yaml_tmp}"
+  sed -i'' -e "s/{{KUBE_CACHE_MUTATION_DETECTOR}}/${KUBE_CACHE_MUTATION_DETECTOR}/g" "${component_yaml_tmp}"
+  kubectl --context="${HOST_CLUSTER_NAME}" apply -f "${component_yaml_tmp}"
+  
+  util::wait_pod_ready "${HOST_CLUSTER_NAME}" "${wait_label}" "${KARMADA_SYSTEM_NAMESPACE}"
+}
+
 # Use x.x.x.8 IP address, which is the same CIDR with the node address of the Kind cluster,
 # as the loadBalancer service address of component karmada-interpreter-webhook-example.
 interpreter_webhook_example_service_external_ip_prefix=$(echo $(util::get_apiserver_ip_from_kubeconfig "${HOST_CLUSTER_NAME}") | awk -F. '{printf "%s.%s.%s",$1,$2,$3}')
@@ -376,15 +392,14 @@ util::append_client_kubeconfig "${HOST_CLUSTER_KUBECONFIG}" "${ROOT_CA_FILE}" "$
 cp "${REPO_ROOT}"/artifacts/deploy/kube-controller-manager.yaml "${TEMP_PATH_APISERVER}"/kube-controller-manager.yaml
 sed -i'' -e "s/{{karmada_apiserver_version}}/${KARMADA_APISERVER_VERSION}/g" "${TEMP_PATH_APISERVER}"/kube-controller-manager.yaml
 kubectl --context="${HOST_CLUSTER_NAME}" apply -f "${TEMP_PATH_APISERVER}"/kube-controller-manager.yaml
+util::wait_pod_ready "${HOST_CLUSTER_NAME}" "${KUBE_CONTROLLER_POD_LABEL}" "${KARMADA_SYSTEM_NAMESPACE}"
 # deploy aggregated-apiserver on host cluster
 kubectl --context="${HOST_CLUSTER_NAME}" apply -f "${REPO_ROOT}/artifacts/deploy/karmada-aggregated-apiserver.yaml"
 util::wait_pod_ready "${HOST_CLUSTER_NAME}" "${KARMADA_AGGREGATION_APISERVER_LABEL}" "${KARMADA_SYSTEM_NAMESPACE}"
-# deploy karmada-search on host cluster
-kubectl --context="${HOST_CLUSTER_NAME}" apply -f "${REPO_ROOT}/artifacts/deploy/karmada-search.yaml"
-util::wait_pod_ready "${HOST_CLUSTER_NAME}" "${KARMADA_SEARCH_LABEL}" "${KARMADA_SYSTEM_NAMESPACE}"
-# deploy karmada-metrics-adapter on host cluster
-kubectl --context="${HOST_CLUSTER_NAME}" apply -f "${REPO_ROOT}/artifacts/deploy/karmada-metrics-adapter.yaml"
-util::wait_pod_ready "${HOST_CLUSTER_NAME}" "${KARMADA_METRICS_ADAPTER_LABEL}" "${KARMADA_SYSTEM_NAMESPACE}"
+
+# Deploy components that need to wait for readiness
+deploy_karmada_component "karmada-search" "${TEMP_PATH_APISERVER}" "${KARMADA_SEARCH_LABEL}"
+deploy_karmada_component "karmada-metrics-adapter" "${TEMP_PATH_APISERVER}" "${KARMADA_METRICS_ADAPTER_LABEL}"
 
 # install CRD APIs on karmada apiserver.
 if ! kubectl config get-contexts "karmada-apiserver" > /dev/null 2>&1;
@@ -449,17 +464,10 @@ sed -i'' -e "s/{{ca_crt}}/${karmada_ca}/g" "${TEMP_PATH_BOOTSTRAP}"/bootstrap-to
 sed -i'' -e "s|{{apiserver_address}}|${karmada_apiserver_address}|g" "${TEMP_PATH_BOOTSTRAP}"/bootstrap-token-configuration-tmp.yaml
 kubectl --context="karmada-apiserver" apply -f "${TEMP_PATH_BOOTSTRAP}"/bootstrap-token-configuration-tmp.yaml
 
-# deploy controller-manager on host cluster
-kubectl --context="${HOST_CLUSTER_NAME}" apply -f "${REPO_ROOT}/artifacts/deploy/karmada-controller-manager.yaml"
-# deploy scheduler on host cluster
-kubectl --context="${HOST_CLUSTER_NAME}" apply -f "${REPO_ROOT}/artifacts/deploy/karmada-scheduler.yaml"
-# deploy descheduler on host cluster
-kubectl --context="${HOST_CLUSTER_NAME}" apply -f "${REPO_ROOT}/artifacts/deploy/karmada-descheduler.yaml"
+deploy_karmada_component "karmada-controller-manager" "${TEMP_PATH_BOOTSTRAP}" "${KARMADA_CONTROLLER_LABEL}"
+deploy_karmada_component "karmada-scheduler" "${TEMP_PATH_BOOTSTRAP}" "${KARMADA_SCHEDULER_LABEL}"
+deploy_karmada_component "karmada-descheduler" "${TEMP_PATH_BOOTSTRAP}" "${KARMADA_DESCHEDULER_LABEL}"
+
 # deploy webhook on host cluster
 kubectl --context="${HOST_CLUSTER_NAME}" apply -f "${REPO_ROOT}/artifacts/deploy/karmada-webhook.yaml"
-
-# make sure all karmada control plane components are ready
-util::wait_pod_ready "${HOST_CLUSTER_NAME}" "${KARMADA_CONTROLLER_LABEL}" "${KARMADA_SYSTEM_NAMESPACE}"
-util::wait_pod_ready "${HOST_CLUSTER_NAME}" "${KARMADA_SCHEDULER_LABEL}" "${KARMADA_SYSTEM_NAMESPACE}"
-util::wait_pod_ready "${HOST_CLUSTER_NAME}" "${KUBE_CONTROLLER_POD_LABEL}" "${KARMADA_SYSTEM_NAMESPACE}"
 util::wait_pod_ready "${HOST_CLUSTER_NAME}" "${KARMADA_WEBHOOK_LABEL}" "${KARMADA_SYSTEM_NAMESPACE}"

hack/deploy-scheduler-estimator.sh

@@ -17,6 +17,9 @@
 set -o errexit
 set -o nounset
 
+# Do not run the mutation detector by default on the local karmada instance.
+KUBE_CACHE_MUTATION_DETECTOR="${KUBE_CACHE_MUTATION_DETECTOR:-false}"
+
 REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 function usage() {
   echo "This script will deploy karmada-scheduler-estimator of a cluster."
@@ -85,6 +88,7 @@ rm -rf "${TEMP_PATH}"
 TEMP_PATH=$(mktemp -d)
 cp "${REPO_ROOT}"/artifacts/deploy/karmada-scheduler-estimator.yaml "${TEMP_PATH}"/karmada-scheduler-estimator.yaml
 sed -i'' -e "s/{{member_cluster_name}}/${MEMBER_CLUSTER_NAME}/g" "${TEMP_PATH}"/karmada-scheduler-estimator.yaml
+sed -i'' -e "s/{{KUBE_CACHE_MUTATION_DETECTOR}}/${KUBE_CACHE_MUTATION_DETECTOR}/g" "${TEMP_PATH}"/karmada-scheduler-estimator.yaml
 echo -e "Apply dynamic rendered deployment in ${TEMP_PATH}/karmada-scheduler-estimator.yaml\n"
 kubectl --kubeconfig="${HOST_CLUSTER_KUBECONFIG}" --context="${HOST_CLUSTER_NAME}" apply -f "${TEMP_PATH}"/karmada-scheduler-estimator.yaml
 rm -rf "${TEMP_PATH}"

hack/run-e2e.sh

@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 set -o errexit
 set -o nounset
 set -o pipefail
@@ -26,17 +25,21 @@ set -o pipefail
 # Example 1: hack/run-e2e.sh (run e2e with default config)
 # Example 2: export KARMADA_APISERVER_KUBECONFIG=<KUBECONFIG PATH> hack/run-e2e.sh (run e2e with your KUBECONFIG)
 
+# Cluster name definitions
+MEMBER_CLUSTER_1_NAME=${MEMBER_CLUSTER_1_NAME:-"member1"}
+MEMBER_CLUSTER_2_NAME=${MEMBER_CLUSTER_2_NAME:-"member2"}
+PULL_MODE_CLUSTER_NAME=${PULL_MODE_CLUSTER_NAME:-"member3"}
+
 KUBECONFIG_PATH=${KUBECONFIG_PATH:-"${HOME}/.kube"}
 KARMADA_APISERVER_KUBECONFIG=${KARMADA_APISERVER_KUBECONFIG:-"$KUBECONFIG_PATH/karmada.config"}
-PULL_BASED_CLUSTERS=${PULL_BASED_CLUSTERS:-"member3:$KUBECONFIG_PATH/members.config"}
+PULL_BASED_CLUSTERS=${PULL_BASED_CLUSTERS:-"${PULL_MODE_CLUSTER_NAME}:$KUBECONFIG_PATH/members.config"}
 
 # KARMADA_RUNNING_ON_KIND indicates if current testing against on karmada that installed on a kind cluster.
 # Defaults to true.
 # For kind cluster, the kind related logs will be collected after the testing.
 KARMADA_RUNNING_ON_KIND=${KARMADA_RUNNING_ON_KIND:-true}
 
 KARMADA_HOST_CLUSTER_NAME=${KARMADA_HOST_CLUSTER_NAME:-"karmada-host"}
-KARMADA_PULL_CLUSTER_NAME=${KARMADA_PULL_CLUSTER_NAME:-"member3"}
 
 ARTIFACTS_PATH=${ARTIFACTS_PATH:-"${HOME}/karmada-e2e-logs"}
 mkdir -p "$ARTIFACTS_PATH"
@@ -65,9 +68,9 @@ if [ "$KARMADA_RUNNING_ON_KIND" = true ]; then
   mkdir -p "$ARTIFACTS_PATH/$KARMADA_HOST_CLUSTER_NAME"
   kind export logs --name="$KARMADA_HOST_CLUSTER_NAME" "$ARTIFACTS_PATH/$KARMADA_HOST_CLUSTER_NAME"
 
-  echo "Collecting $KARMADA_PULL_CLUSTER_NAME logs..."
-  mkdir -p "$ARTIFACTS_PATH/KARMADA_PULL_CLUSTER_NAME"
-  kind export logs --name="$KARMADA_PULL_CLUSTER_NAME" "$ARTIFACTS_PATH/$KARMADA_PULL_CLUSTER_NAME"
+  echo "Collecting $PULL_MODE_CLUSTER_NAME logs..."
+  mkdir -p "$ARTIFACTS_PATH/$PULL_MODE_CLUSTER_NAME"
+  kind export logs --name="$PULL_MODE_CLUSTER_NAME" "$ARTIFACTS_PATH/$PULL_MODE_CLUSTER_NAME"
 fi
 
 echo "Collected logs at $ARTIFACTS_PATH:"
@@ -76,4 +79,78 @@ ls -al "$ARTIFACTS_PATH"
 # Post run e2e for delete extra components
 "${REPO_ROOT}"/hack/post-run-e2e.sh
 
+# If E2E test failed, exit directly with the test result
+if [ $TESTING_RESULT -ne 0 ]; then
+  echo "E2E test failed with exit code $TESTING_RESULT, skipping component restart check."
+  exit $TESTING_RESULT
+fi
+
+# Check if Karmada components have restarted, if any has, it means that OOM or panic has occurred
+# due to memory modification, and needs to be investigated.
+echo "E2E run successfully."
+echo "Checking if Karmada components have restarted..."
+
+# Function to check pod restart count for a given component
+check_component_restart() {
+  local component_label=$1
+  local component_name=$2
+  
+  echo "Checking ${component_name} pods..."
+  
+  # Get pod information in a single call, including both name and restart count
+  # Use a template that handles missing containerStatuses gracefully
+  local pod_info
+  pod_info=$(kubectl --context="${KARMADA_HOST_CLUSTER_NAME}" get pod -n karmada-system -l "${component_label}" \
+    -o go-template='{{range .items}}{{.metadata.name}}:{{if .status.containerStatuses}}{{(index .status.containerStatuses 0).restartCount}}{{else}}0{{end}}{{"\n"}}{{end}}' 2>/dev/null)
+  
+  if [ -z "$pod_info" ]; then
+    echo "No pods found for ${component_name}, skipping..."
+    return 0
+  fi
+  
+  # Process each pod's information
+  while IFS=: read -r pod_name restart_count; do
+    # Skip empty lines
+    [ -z "$pod_name" ] && continue
+    
+    # Ensure restart_count is a number (default to 0 if empty or invalid)
+    if ! [[ "$restart_count" =~ ^[0-9]+$ ]]; then
+      echo "Warning: Unable to get restart count for pod $pod_name, assuming 0"
+      restart_count=0
+    fi
+    
+    if [ "$restart_count" -gt 0 ]; then
+      echo "ERROR: ${component_name} pod $pod_name has restarted $restart_count times."
+      echo "This indicates OOM or panic occurred and needs to be investigated."
+      return 1  # Return failure to stop checking
+    else
+      echo "${component_name} pod $pod_name: no restarts"
+    fi
+  done <<< "$pod_info"
+  
+  return 0
+}
+
+# List of components to check (label=component_name)
+components=(
+  "app=karmada-controller-manager:karmada-controller-manager"
+  "app=karmada-descheduler:karmada-descheduler"
+  "app=karmada-metrics-adapter:karmada-metrics-adapter"
+  "app=karmada-scheduler:karmada-scheduler"
+  "app=karmada-search:karmada-search"
+  "app=karmada-scheduler-estimator-${MEMBER_CLUSTER_1_NAME}:karmada-scheduler-estimator-${MEMBER_CLUSTER_1_NAME}"
+  "app=karmada-scheduler-estimator-${MEMBER_CLUSTER_2_NAME}:karmada-scheduler-estimator-${MEMBER_CLUSTER_2_NAME}"
+  "app=karmada-scheduler-estimator-${PULL_MODE_CLUSTER_NAME}:karmada-scheduler-estimator-${PULL_MODE_CLUSTER_NAME}"
+)
+
+# Check each component, stop at first failure
+for component in "${components[@]}"; do
+  IFS=':' read -r label name <<< "$component"
+  if ! check_component_restart "$label" "$name"; then
+    echo "COMPONENT RESTART CHECK FAILED: Component $name has restarted, stopping further checks."
+    exit 1
+  fi
+done
+
+echo "All component restart checks passed."
 exit $TESTING_RESULT