Ability to set up additional, bigger nodes during tests

Author: jkaniuk

cluster/gce/config-default.sh

@@ -180,6 +180,13 @@ ENABLE_METADATA_AGENT="${KUBE_ENABLE_METADATA_AGENT:-none}"
 # Useful for scheduling heapster in large clusters with nodes of small size.
 HEAPSTER_MACHINE_TYPE="${HEAPSTER_MACHINE_TYPE:-}"
 
+# Optional: Additional nodes would be created if their type and number is specified.
+# NUM_NODES would be lowered respectively.
+# Useful for running cluster-level addons that needs more resources than would fit
+# on small nodes, like network plugins.
+NUM_ADDITIONAL_NODES="${NUM_ADDITIONAL_NODES:-}"
+ADDITIONAL_MACHINE_TYPE="${ADDITIONAL_MACHINE_TYPE:-}"
+
 MASTER_NODE_LABELS="${KUBE_MASTER_NODE_LABELS:-}"
 # NON_MASTER_NODE_LABELS are labels will only be applied on non-master nodes.
 NON_MASTER_NODE_LABELS="${KUBE_NON_MASTER_NODE_LABELS:-}"

cluster/gce/config-test.sh

@@ -192,6 +192,13 @@ ENABLE_METADATA_AGENT="${KUBE_ENABLE_METADATA_AGENT:-none}"
 # Useful for scheduling heapster in large clusters with nodes of small size.
 HEAPSTER_MACHINE_TYPE="${HEAPSTER_MACHINE_TYPE:-}"
 
+# Optional: Additional nodes would be created if their type and number is specified.
+# NUM_NODES would be lowered respectively.
+# Useful for running cluster-level addons that needs more resources than would fit
+# on small nodes, like network plugins.
+NUM_ADDITIONAL_NODES="${NUM_ADDITIONAL_NODES:-}"
+ADDITIONAL_MACHINE_TYPE="${ADDITIONAL_MACHINE_TYPE:-}"
+
 # Set etcd image (e.g. k8s.gcr.io/etcd) and version (e.g. 3.3.15-0) if you need
 # non-default version.
 ETCD_IMAGE="${TEST_ETCD_IMAGE:-}"

cluster/gce/gci/node-helper.sh

@@ -39,7 +39,9 @@ function get-node-instance-metadata-from-file {
 #   $1: template name (required).
 function create-linux-node-instance-template {
   local template_name="$1"
+  local machine_type="${2:-$NODE_SIZE}"
+
   ensure-gci-metadata-files
   # shellcheck disable=2154 # 'scope_flags' is assigned by upstream
-  create-node-template "${template_name}" "${scope_flags[*]}" "$(get-node-instance-metadata-from-file)" "" "linux"
+  create-node-template "${template_name}" "${scope_flags[*]}" "$(get-node-instance-metadata-from-file)" "" "linux" "${machine_type}"
 }

cluster/gce/util.sh

@@ -2130,6 +2130,7 @@ function create-node-template() {
   local template_name="$1"
   local metadata_values="$4"
   local os="$5"
+  local machine_type="$6"
 
   # First, ensure the template doesn't exist.
   # TODO(zmerlynn): To make this really robust, we need to parse the output and
@@ -2216,7 +2217,7 @@ function create-node-template() {
     if ! ${gcloud} compute instance-templates create \
       "${template_name}" \
       --project "${PROJECT}" \
-      --machine-type "${NODE_SIZE}" \
+      --machine-type "${machine_type}" \
       --boot-disk-type "${NODE_DISK_TYPE}" \
       --boot-disk-size "${NODE_DISK_SIZE}" \
       ${node_image_flags} \
@@ -3068,6 +3069,10 @@ function create-nodes-template() {
   local windows_template_name="${WINDOWS_NODE_INSTANCE_PREFIX}-template"
   create-linux-node-instance-template $linux_template_name
   create-windows-node-instance-template $windows_template_name "${scope_flags[*]}"
+  if [[ -n "${ADDITIONAL_MACHINE_TYPE:-}" ]]; then
+    local linux_extra_template_name="${NODE_INSTANCE_PREFIX}-extra-template"
+    create-linux-node-instance-template $linux_extra_template_name "${ADDITIONAL_MACHINE_TYPE}"
+  fi
 }
 
 # Assumes:
@@ -3096,13 +3101,38 @@ function set_num_migs() {
 # - ZONE
 function create-linux-nodes() {
   local template_name="${NODE_INSTANCE_PREFIX}-template"
+  local extra_template_name="${NODE_INSTANCE_PREFIX}-extra-template"
 
-  if [[ -z "${HEAPSTER_MACHINE_TYPE:-}" ]]; then
-    local -r nodes="${NUM_NODES}"
-  else
+  local nodes="${NUM_NODES}"
+  if [[ ! -z "${HEAPSTER_MACHINE_TYPE:-}" ]]; then
     echo "Creating a special node for heapster with machine-type ${HEAPSTER_MACHINE_TYPE}"
     create-heapster-node
-    local -r nodes=$(( NUM_NODES - 1 ))
+    nodes=$(( nodes - 1 ))
+  fi
+
+  if [[ -n "${ADDITIONAL_MACHINE_TYPE:-}" && "${NUM_ADDITIONAL_NODES:-}" -gt 0 ]]; then
+    local num_additional="${NUM_ADDITIONAL_NODES}"
+    if [[ "${NUM_ADDITIONAL_NODES:-}" -gt "${nodes}" ]]; then
+      echo "Capping NUM_ADDITIONAL_NODES to ${nodes}"
+      num_additional="${nodes}"
+    fi
+    if [[ "${num_additional:-}" -gt 0 ]]; then
+      echo "Creating ${num_additional} special nodes with machine-type ${ADDITIONAL_MACHINE_TYPE}"
+      local extra_group_name="${NODE_INSTANCE_PREFIX}-extra"
+      gcloud compute instance-groups managed \
+          create "${extra_group_name}" \
+          --project "${PROJECT}" \
+          --zone "${ZONE}" \
+          --base-instance-name "${extra_group_name}" \
+          --size "${num_additional}" \
+          --template "${extra_template_name}" || true;
+      gcloud compute instance-groups managed wait-until-stable \
+          "${extra_group_name}" \
+          --zone "${ZONE}" \
+          --project "${PROJECT}" \
+          --timeout "${MIG_WAIT_UNTIL_STABLE_TIMEOUT}" || true
+      nodes=$(( nodes - $num_additional ))
+    fi
   fi
 
   local instances_left=${nodes}
@@ -3414,13 +3444,15 @@ function kube-down() {
 
     local all_instance_groups=(${INSTANCE_GROUPS[@]:-} ${WINDOWS_INSTANCE_GROUPS[@]:-})
     for group in ${all_instance_groups[@]:-}; do
-      if gcloud compute instance-groups managed describe "${group}" --project "${PROJECT}" --zone "${ZONE}" &>/dev/null; then
-        gcloud compute instance-groups managed delete \
-          --project "${PROJECT}" \
-          --quiet \
-          --zone "${ZONE}" \
-          "${group}" &
-      fi
+      {
+        if gcloud compute instance-groups managed describe "${group}" --project "${PROJECT}" --zone "${ZONE}" &>/dev/null; then
+          gcloud compute instance-groups managed delete \
+            --project "${PROJECT}" \
+            --quiet \
+            --zone "${ZONE}" \
+            "${group}"
+        fi
+      } &
     done
 
     # Wait for last batch of jobs
@@ -3429,14 +3461,21 @@ function kube-down() {
     }
 
     for template in ${templates[@]:-}; do
-      if gcloud compute instance-templates describe --project "${PROJECT}" "${template}" &>/dev/null; then
-        gcloud compute instance-templates delete \
-          --project "${PROJECT}" \
-          --quiet \
-          "${template}"
-      fi
+      {
+        if gcloud compute instance-templates describe --project "${PROJECT}" "${template}" &>/dev/null; then
+          gcloud compute instance-templates delete \
+            --project "${PROJECT}" \
+            --quiet \
+            "${template}"
+        fi
+      } &
     done
 
+    # Wait for last batch of jobs
+    kube::util::wait-for-jobs || {
+      echo -e "Failed to delete instance template(s)." >&2
+    }
+
     # Delete the special heapster node (if it exists).
     if [[ -n "${HEAPSTER_MACHINE_TYPE:-}" ]]; then
       local -r heapster_machine_name="${NODE_INSTANCE_PREFIX}-heapster"
@@ -3717,7 +3756,7 @@ function set-replica-name() {
 #
 # $1: project
 function get-template() {
-  local linux_filter="${NODE_INSTANCE_PREFIX}-template(-(${KUBE_RELEASE_VERSION_DASHED_REGEX}|${KUBE_CI_VERSION_DASHED_REGEX}))?"
+  local linux_filter="${NODE_INSTANCE_PREFIX}-(extra-)?template(-(${KUBE_RELEASE_VERSION_DASHED_REGEX}|${KUBE_CI_VERSION_DASHED_REGEX}))?"
   local windows_filter="${WINDOWS_NODE_INSTANCE_PREFIX}-template(-(${KUBE_RELEASE_VERSION_DASHED_REGEX}|${KUBE_CI_VERSION_DASHED_REGEX}))?"
 
   gcloud compute instance-templates list \

cluster/gce/windows/node-helper.sh

@@ -54,5 +54,5 @@ function get-windows-node-instance-metadata {
 function create-windows-node-instance-template {
   local template_name="$1"
   local scopes_flag="$2"
-  create-node-template "${template_name}" "${scopes_flag}" "$(get-windows-node-instance-metadata-from-file)" "$(get-windows-node-instance-metadata)" "windows"
+  create-node-template "${template_name}" "${scopes_flag}" "$(get-windows-node-instance-metadata-from-file)" "$(get-windows-node-instance-metadata)" "windows" "${NODE_SIZE}"
 }