Merge pull request kubernetes#95312 from benhxy/gke/etcd-ip

Make GKE kube-apiserver use host IP to connect to etcd

Author: k8s-ci-robot

cluster/gce/config-default.sh

@@ -524,3 +524,8 @@ export KONNECTIVITY_SERVICE_PROXY_PROTOCOL_MODE="${KUBE_KONNECTIVITY_SERVICE_PRO
 
 # Optional: Enable Windows CSI-Proxy
 export ENABLE_CSI_PROXY="${ENABLE_CSI_PROXY:-true}"
+
+# ETCD_LISTEN_ON_HOST_IP decides whether etcd servers should also listen on host IP, 
+# in addition to listening to 127.0.0.1, and whether kube-apiserver should connect to etcd servers
+# through host IP.
+export ETCD_LISTEN_ON_HOST_IP="${ETCD_LISTEN_ON_HOST_IP:-false}"

cluster/gce/config-test.sh

@@ -571,3 +571,8 @@ export GCE_UPLOAD_KUBCONFIG_TO_MASTER_METADATA=true
 
 # Optoinal: Enable Windows CSI-Proxy
 export ENABLE_CSI_PROXY="${ENABLE_CSI_PROXY:-true}"
+
+# ETCD_LISTEN_ON_HOST_IP decides whether etcd servers should also listen on host IP, 
+# in addition to listening to 127.0.0.1, and whether kube-apiserver should connect to etcd servers
+# through host IP.
+export ETCD_LISTEN_ON_HOST_IP="${ETCD_LISTEN_ON_HOST_IP:-false}"

cluster/gce/gci/apiserver_etcd_test.go

@@ -37,6 +37,8 @@ type kubeAPIServeETCDEnv struct {
 	StorageBackend      string
 	StorageMediaType    string
 	CompactionInterval  string
+	HostPrimaryIP       string
+	ETCDListenOnHostIP  string
 }
 
 func TestServerOverride(t *testing.T) {
@@ -52,7 +54,7 @@ func TestServerOverride(t *testing.T) {
 			},
 		},
 		{
-			desc: "ETCD-SERVERS and ETCD_SERVERS_OVERRIDES iare set",
+			desc: "ETCD-SERVERS and ETCD_SERVERS_OVERRIDES are set",
 			env: kubeAPIServeETCDEnv{
 				ETCDServers:         "ETCDServers",
 				ETCDServersOverride: "ETCDServersOverrides",
@@ -61,6 +63,17 @@ func TestServerOverride(t *testing.T) {
 				"--etcd-servers-overrides=ETCDServersOverrides",
 			},
 		},
+		{
+			desc: "HOST_PRIMARY_IP is set and etcd is set to listen to host IP",
+			env: kubeAPIServeETCDEnv{
+				HostPrimaryIP:      "HostPrimaryIP",
+				ETCDListenOnHostIP: "true",
+			},
+			want: []string{
+				"--etcd-servers-overrides=/events#http://HostPrimaryIP:4002",
+				"--etcd-servers=http://HostPrimaryIP:2379",
+			},
+		},
 	}
 
 	for _, tc := range testCases {

cluster/gce/gci/configure-helper.sh

@@ -1686,8 +1686,8 @@ function start-kube-proxy {
 # Replaces the variables in the etcd manifest file with the real values, and then
 # copy the file to the manifest dir
 # $1: value for variable 'suffix'
-# $2: value for variable 'port'
-# $3: value for variable 'server_port'
+# $2: value for variable 'port', for listening to clients
+# $3: value for variable 'server_port', for etcd peering
 # $4: value for variable 'cpulimit'
 # $5: pod name, which should be either etcd or etcd-events
 function prepare-etcd-manifest {
@@ -1706,19 +1706,39 @@ function prepare-etcd-manifest {
   if [[ -n "${INITIAL_ETCD_CLUSTER_STATE:-}" ]]; then
     cluster_state="${INITIAL_ETCD_CLUSTER_STATE}"
   fi
+
+  # Configure mTLS for etcd peers.
   if [[ -n "${ETCD_CA_CERT:-}" && -n "${ETCD_PEER_KEY:-}" && -n "${ETCD_PEER_CERT:-}" ]]; then
     etcd_creds=" --peer-trusted-ca-file /etc/srv/kubernetes/etcd-ca.crt --peer-cert-file /etc/srv/kubernetes/etcd-peer.crt --peer-key-file /etc/srv/kubernetes/etcd-peer.key -peer-client-cert-auth "
     etcd_protocol="https"
   fi
 
-  # mTLS should only be enabled for etcd server but not etcd-events. if $1 suffix is empty, it's etcd server.
+  # host_primary_ip is the primary internal IP of the host.
+  # Override host primary IP if specifically provided.
+  local host_primary_ip
+  host_primary_ip="${HOST_PRIMARY_IP:-$(hostname -i)}"
+
+  # Configure mTLS for clients (e.g. kube-apiserver).
+  # mTLS should only be enabled for etcd server but not etcd-events. If $1 suffix is empty, it's etcd server.
+  local etcd_listen_metrics_urls=""
   if [[ -z "${suffix}" && -n "${ETCD_APISERVER_CA_KEY:-}" && -n "${ETCD_APISERVER_CA_CERT:-}" && -n "${ETCD_APISERVER_SERVER_KEY:-}" && -n "${ETCD_APISERVER_SERVER_CERT:-}" && -n "${ETCD_APISERVER_CLIENT_KEY:-}" && -n "${ETCD_APISERVER_CLIENT_CERT:-}" ]]; then
     etcd_apiserver_creds=" --client-cert-auth --trusted-ca-file ${ETCD_APISERVER_CA_CERT_PATH} --cert-file ${ETCD_APISERVER_SERVER_CERT_PATH} --key-file ${ETCD_APISERVER_SERVER_KEY_PATH} "
     etcd_apiserver_protocol="https"
     etcd_livenessprobe_port="2382"
-    etcd_extra_args+=" --listen-metrics-urls=http://${ETCD_LISTEN_CLIENT_IP:-127.0.0.1}:${etcd_livenessprobe_port} "
+    etcd_listen_metrics_urls="http://${ETCD_LISTEN_CLIENT_IP:-127.0.0.1}:${etcd_livenessprobe_port}"
+    if [[ ${ETCD_LISTEN_ON_HOST_IP:-} == "true" ]]; then
+      etcd_listen_metrics_urls+=",http://${host_primary_ip}:${etcd_livenessprobe_port}"
+    fi
+    etcd_extra_args+=" --listen-metrics-urls=${etcd_listen_metrics_urls} "
+  fi
+
+  # If etcd is configured to listen on host IP, an additional client listening URL is added.
+  local etcd_listen_client_urls="${etcd_apiserver_protocol}://${ETCD_LISTEN_CLIENT_IP:-127.0.0.1}:$2"
+  if [[ ${ETCD_LISTEN_ON_HOST_IP:-} == "true" ]] ; then
+    etcd_listen_client_urls+=",${etcd_apiserver_protocol}://${host_primary_ip}:$2"
   fi
 
+  # Generate etcd member URLs.
   for host in $(echo "${INITIAL_ETCD_CLUSTER:-${host_name}}" | tr "," "\n"); do
     etcd_host="etcd-${host}=${etcd_protocol}://${host}:$3"
     if [[ -n "${etcd_cluster}" ]]; then
@@ -1738,6 +1758,7 @@ function prepare-etcd-manifest {
   sed -i -e "s@{{ *etcd_cluster *}}@$etcd_cluster@g" "${temp_file}"
   sed -i -e "s@{{ *liveness_probe_initial_delay *}}@${ETCD_LIVENESS_PROBE_INITIAL_DELAY_SEC:-15}@g" "${temp_file}"
   sed -i -e "s@{{ *listen_client_ip *}}@${ETCD_LISTEN_CLIENT_IP:-127.0.0.1}@g" "${temp_file}"
+  sed -i -e "s@{{ *etcd_listen_client_urls *}}@${etcd_listen_client_urls:-}@g" "${temp_file}"
   # Get default storage backend from manifest file.
   local -r default_storage_backend=$( \
     grep -o "{{ *pillar\.get('storage_backend', '\(.*\)') *}}" "${temp_file}" | \

cluster/gce/gci/configure-kubeapiserver.sh

@@ -14,25 +14,34 @@
 # limitations under the License.
 
 
-# Configures etcd related flags of kube-apiserver.
+# Configures etcd related parameters of kube-apiserver.
 function configure-etcd-params {
   local -n params_ref=$1
 
+  local host_ip="127.0.0.1"
+  # If etcd is configured to listen on host IP,
+  # host_ip is set to the primary internal IP of host VM.
+  if [[ ${ETCD_LISTEN_ON_HOST_IP:-} == "true" ]] ; then
+      host_ip="${HOST_PRIMARY_IP:-$(hostname -i)}"
+  fi
+
+  # Configure the main etcd.
   if [[ -n "${ETCD_APISERVER_CA_KEY:-}" && -n "${ETCD_APISERVER_CA_CERT:-}" && -n "${ETCD_APISERVER_SERVER_KEY:-}" && -n "${ETCD_APISERVER_SERVER_CERT:-}" && -n "${ETCD_APISERVER_CLIENT_KEY:-}" && -n "${ETCD_APISERVER_CLIENT_CERT:-}" ]]; then
-      params_ref+=" --etcd-servers=${ETCD_SERVERS:-https://127.0.0.1:2379}"
+      params_ref+=" --etcd-servers=${ETCD_SERVERS:-https://${host_ip}:2379}"
       params_ref+=" --etcd-cafile=${ETCD_APISERVER_CA_CERT_PATH}"
       params_ref+=" --etcd-certfile=${ETCD_APISERVER_CLIENT_CERT_PATH}"
       params_ref+=" --etcd-keyfile=${ETCD_APISERVER_CLIENT_KEY_PATH}"
   elif [[ -z "${ETCD_APISERVER_CA_KEY:-}" && -z "${ETCD_APISERVER_CA_CERT:-}" && -z "${ETCD_APISERVER_SERVER_KEY:-}" && -z "${ETCD_APISERVER_SERVER_CERT:-}" && -z "${ETCD_APISERVER_CLIENT_KEY:-}" && -z "${ETCD_APISERVER_CLIENT_CERT:-}" ]]; then
-      params_ref+=" --etcd-servers=${ETCD_SERVERS:-http://127.0.0.1:2379}"
+      params_ref+=" --etcd-servers=${ETCD_SERVERS:-http://${host_ip}:2379}"
       echo "WARNING: ALL of ETCD_APISERVER_CA_KEY, ETCD_APISERVER_CA_CERT, ETCD_APISERVER_SERVER_KEY, ETCD_APISERVER_SERVER_CERT, ETCD_APISERVER_CLIENT_KEY and ETCD_APISERVER_CLIENT_CERT are missing, mTLS between etcd server and kube-apiserver is not enabled."
   else
       echo "ERROR: Some of ETCD_APISERVER_CA_KEY, ETCD_APISERVER_CA_CERT, ETCD_APISERVER_SERVER_KEY, ETCD_APISERVER_SERVER_CERT, ETCD_APISERVER_CLIENT_KEY and ETCD_APISERVER_CLIENT_CERT are missing, mTLS between etcd server and kube-apiserver cannot be enabled. Please provide all mTLS credential."
       exit 1
   fi
 
+  # Configure the event log etcd.
   if [[ -z "${ETCD_SERVERS:-}" ]]; then
-    params_ref+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-/events#http://127.0.0.1:4002}"
+    params_ref+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-/events#http://${host_ip}:4002}"
   elif [[ -n "${ETCD_SERVERS_OVERRIDES:-}" ]]; then
     params_ref+=" --etcd-servers-overrides=${ETCD_SERVERS_OVERRIDES:-}"
   fi

cluster/gce/gci/testdata/kube-apiserver/etcd.template

@@ -13,3 +13,5 @@ readonly ETCD_SERVERS_OVERRIDES={{.ETCDServersOverride}}
 readonly STORAGE_BACKEND={{.StorageBackend}}
 readonly STORAGE_MEDIA_TYPE={{.StorageMediaType}}
 readonly ETCD_COMPACTION_INTERVAL_SEC={{.CompactionInterval}}
+readonly HOST_PRIMARY_IP={{.HostPrimaryIP}}
+readonly ETCD_LISTEN_ON_HOST_IP={{.ETCDListenOnHostIP}}

cluster/gce/manifests/etcd.manifest

@@ -26,7 +26,7 @@
     "command": [
               "/bin/sh",
               "-c",
-              "set -o errexit; if [ -e /usr/local/bin/migrate-if-needed.sh ]; then /usr/local/bin/migrate-if-needed.sh 1>>/var/log/etcd{{ suffix }}.log 2>&1; fi; exec /usr/local/bin/etcd --name etcd-{{ hostname }} --listen-peer-urls {{ etcd_protocol }}://{{ host_ip }}:{{ server_port }} --initial-advertise-peer-urls {{ etcd_protocol }}://{{ hostname }}:{{ server_port }} --advertise-client-urls {{ etcd_apiserver_protocol }}://127.0.0.1:{{ port }} --listen-client-urls {{ etcd_apiserver_protocol }}://{{ listen_client_ip }}:{{ port }} {{ quota_bytes }} --data-dir /var/etcd/data{{ suffix }} --initial-cluster-state {{ cluster_state }} --initial-cluster {{ etcd_cluster }} {{ etcd_creds }} {{ etcd_apiserver_creds }} {{ etcd_extra_args }} 1>>/var/log/etcd{{ suffix }}.log 2>&1"
+              "set -o errexit; if [ -e /usr/local/bin/migrate-if-needed.sh ]; then /usr/local/bin/migrate-if-needed.sh 1>>/var/log/etcd{{ suffix }}.log 2>&1; fi; exec /usr/local/bin/etcd --name etcd-{{ hostname }} --listen-peer-urls {{ etcd_protocol }}://{{ host_ip }}:{{ server_port }} --initial-advertise-peer-urls {{ etcd_protocol }}://{{ hostname }}:{{ server_port }} --advertise-client-urls {{ etcd_apiserver_protocol }}://127.0.0.1:{{ port }} --listen-client-urls {{ etcd_listen_client_urls }} {{ quota_bytes }} --data-dir /var/etcd/data{{ suffix }} --initial-cluster-state {{ cluster_state }} --initial-cluster {{ etcd_cluster }} {{ etcd_creds }} {{ etcd_apiserver_creds }} {{ etcd_extra_args }} 1>>/var/log/etcd{{ suffix }}.log 2>&1"
             ],
     "env": [
       { "name": "TARGET_STORAGE",
@@ -58,8 +58,11 @@
       },
       { "name": "ETCD_HOSTNAME",
         "value": "{{ hostname }}"
+      },
+      { "name": "LISTEN_CLIENT_URLS",
+        "value": "{{ etcd_listen_client_urls }}"
       }
-        ],
+    ],
     "livenessProbe": {
       "httpGet": {
         "host": "127.0.0.1",