Skip to content
This repository was archived by the owner on Jul 30, 2021. It is now read-only.

Commit 769b540

Browse files
xiang90xiang90
authored
Merge pull request #592 from hongchaodeng/t
selfHostedEtcd: enable TLS
2 parents 0f487dc + 23c54ab commit 769b540

File tree

7 files changed

+85
-39
lines changed

7 files changed

+85
-39
lines changed

‎cmd/bootkube/render.go‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,12 @@ import (
1818
)
1919

2020
const (
21-
apiOffset = 1
22-
dnsOffset = 10
23-
etcdOffset = 15
24-
bootEtcdOffset = 20
25-
defaultServiceBaseIP = "10.3.0.0"
26-
defaultEtcdServers = "https://127.0.0.1:2379"
27-
defaultSelfHostedEtcdServers = "http://127.0.0.1:2379"
21+
apiOffset = 1
22+
dnsOffset = 10
23+
etcdOffset = 15
24+
bootEtcdOffset = 20
25+
defaultServiceBaseIP = "10.3.0.0"
26+
defaultEtcdServers = "https://127.0.0.1:2379"
2827
)
2928

3029
var (
@@ -176,13 +175,13 @@ func flagsToAssetConfig() (c *asset.Config, err error) {
176175

177176
var etcdServers []*url.URL
178177
if renderOpts.selfHostedEtcd {
179-
etcdServerUrl, err := url.Parse(fmt.Sprintf("http://%s:2379", etcdServiceIP))
178+
etcdServerUrl, err := url.Parse(fmt.Sprintf("https://%s:2379", etcdServiceIP))
180179
if err != nil {
181180
return nil, err
182181
}
183182
etcdServers = append(etcdServers, etcdServerUrl)
184-
if renderOpts.etcdServers != defaultSelfHostedEtcdServers {
185-
bootkube.UserOutput("--experimental-self-hosted-etcd and --service-cidr set. Overriding --etcd-servers setting with %s\n", etcdServers)
183+
if renderOpts.etcdServers != defaultEtcdServers {
184+
bootkube.UserOutput("--experimental-self-hosted-etcd and --service-cidr set. Overriding --etcd-servers setting (%s) with (%s) \n", etcdServers, defaultEtcdServers)
186185
}
187186
} else {
188187
etcdServers, err = parseURLs(renderOpts.etcdServers)

‎pkg/asset/internal/templates.go‎

Lines changed: 32 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -250,7 +250,7 @@ spec:
250250
- --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt
251251
- --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key
252252
{{- end }}
253-
- --etcd-servers={{ range $i, $e := .EtcdServers }}{{ if $i }},{{end}}{{ $e }}{{end}}{{ if .SelfHostedEtcd }},http://127.0.0.1:12379{{end}}
253+
- --etcd-servers={{ range $i, $e := .EtcdServers }}{{ if $i }},{{end}}{{ $e }}{{end}}{{ if .SelfHostedEtcd }},https://127.0.0.1:12379{{end}}
254254
- --insecure-port=0
255255
- --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt
256256
- --kubelet-client-key=/etc/kubernetes/secrets/apiserver.key
@@ -955,14 +955,30 @@ spec:
955955
command:
956956
- /usr/local/bin/etcd
957957
- --name=boot-etcd
958-
- --listen-client-urls=http://0.0.0.0:12379
959-
- --listen-peer-urls=http://0.0.0.0:12380
960-
- --advertise-client-urls=http://{{ .BootEtcdServiceIP }}:12379
961-
- --initial-advertise-peer-urls=http://{{ .BootEtcdServiceIP }}:12380
962-
- --initial-cluster=boot-etcd=http://{{ .BootEtcdServiceIP }}:12380
958+
- --listen-client-urls=https://0.0.0.0:12379
959+
- --listen-peer-urls=https://0.0.0.0:12380
960+
- --advertise-client-urls=https://{{ .BootEtcdServiceIP }}:12379
961+
- --initial-advertise-peer-urls=https://{{ .BootEtcdServiceIP }}:12380
962+
- --initial-cluster=boot-etcd=https://{{ .BootEtcdServiceIP }}:12380
963963
- --initial-cluster-token=bootkube
964964
- --initial-cluster-state=new
965965
- --data-dir=/var/etcd/data
966+
- --peer-client-cert-auth=true
967+
- --peer-trusted-ca-file=/etc/kubernetes/secrets/etcdMember/peer-ca-crt.pem
968+
- --peer-cert-file=/etc/kubernetes/secrets/etcdMember/peer-crt.pem
969+
- --peer-key-file=/etc/kubernetes/secrets/etcdMember/peer-key.pem
970+
- --client-cert-auth=true
971+
- --trusted-ca-file=/etc/kubernetes/secrets/etcdMember/client-ca-crt.pem
972+
- --cert-file=/etc/kubernetes/secrets/etcdMember/client-crt.pem
973+
- --key-file=/etc/kubernetes/secrets/etcdMember/client-key.pem
974+
volumeMounts:
975+
- mountPath: /etc/kubernetes/secrets
976+
name: secrets
977+
readOnly: true
978+
volumes:
979+
- name: secrets
980+
hostPath:
981+
path: /etc/kubernetes/{{ .BootstrapSecretsSubdir }}
966982
hostNetwork: true
967983
restartPolicy: Never
968984
dnsPolicy: ClusterFirstWithHostNet
@@ -1018,7 +1034,16 @@ var EtcdTPRTemplate = []byte(`{
10181034
]
10191035
},
10201036
"selfHosted": {
1021-
"bootMemberClientEndpoint": "http://{{ .BootEtcdServiceIP }}:12379"
1037+
"bootMemberClientEndpoint": "https://{{ .BootEtcdServiceIP }}:12379"
1038+
},
1039+
"TLS": {
1040+
"static": {
1041+
"member": {
1042+
"peerSecret": "etcd-member-peer-tls",
1043+
"clientSecret": "etcd-member-client-tls"
1044+
},
1045+
"operatorSecret": "etcd-operator-client-tls"
1046+
}
10221047
}
10231048
}
10241049
}`)

‎pkg/asset/k8s.go‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,13 @@ const (
1515
// The name of the k8s service that selects self-hosted etcd pods
1616
EtcdServiceName = "etcd-service"
1717

18-
secretNamespace = "kube-system"
19-
secretAPIServerName = "kube-apiserver"
20-
secretCMName = "kube-controller-manager"
21-
secretEtcdMemberPeer = "etcd-member-peer-tls"
22-
secretEtcdMemberCli = "etcd-member-client-tls"
23-
secretEtcdOperator = "etcd-operator-client-tls"
18+
SecretEtcdMemberPeer = "etcd-member-peer-tls"
19+
SecretEtcdMemberCli = "etcd-member-client-tls"
20+
SecretEtcdOperator = "etcd-operator-client-tls"
21+
22+
secretNamespace = "kube-system"
23+
secretAPIServerName = "kube-apiserver"
24+
secretCMName = "kube-controller-manager"
2425
)
2526

2627
type staticConfig struct {
@@ -110,7 +111,7 @@ func newKubeConfigAsset(assets Assets, conf Config) (Asset, error) {
110111
func newSelfHostedEtcdSecretAssets(assets Assets) (Assets, error) {
111112
var res Assets
112113

113-
secretYAML, err := secretFromAssets(secretEtcdMemberPeer, secretNamespace, []string{
114+
secretYAML, err := secretFromAssets(SecretEtcdMemberPeer, secretNamespace, []string{
114115
AssetPathSelfHostedEtcdMemberPeerCA,
115116
AssetPathSelfHostedEtcdMemberPeerCert,
116117
AssetPathSelfHostedEtcdMemberPeerKey,
@@ -120,7 +121,7 @@ func newSelfHostedEtcdSecretAssets(assets Assets) (Assets, error) {
120121
}
121122
res = append(res, Asset{Name: AssetPathSelfHostedEtcdMemberPeerSecret, Data: secretYAML})
122123

123-
secretYAML, err = secretFromAssets(secretEtcdMemberCli, secretNamespace, []string{
124+
secretYAML, err = secretFromAssets(SecretEtcdMemberCli, secretNamespace, []string{
124125
AssetPathSelfHostedEtcdMemberClientCA,
125126
AssetPathSelfHostedEtcdMemberClientCert,
126127
AssetPathSelfHostedEtcdMemberClientKey,
@@ -130,7 +131,7 @@ func newSelfHostedEtcdSecretAssets(assets Assets) (Assets, error) {
130131
}
131132
res = append(res, Asset{Name: AssetPathSelfHostedEtcdMemberCliSecret, Data: secretYAML})
132133

133-
secretYAML, err = secretFromAssets(secretEtcdOperator, secretNamespace, []string{
134+
secretYAML, err = secretFromAssets(SecretEtcdOperator, secretNamespace, []string{
134135
AssetPathSelfHostedOperatorEtcdCA,
135136
AssetPathSelfHostedOperatorEtcdCert,
136137
AssetPathSelfHostedOperatorEtcdKey,

‎pkg/recovery/etcd_template.go‎

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -75,9 +75,9 @@ spec:
7575
/var/etcd-backupdir/{{ .BackupFile }} \
7676
--data-dir=/var/etcd/data \
7777
--name=boot-etcd \
78-
--initial-cluster=boot-etcd=http://{{ .BootEtcdServiceIP }}:12380 \
78+
--initial-cluster=boot-etcd=https://{{ .BootEtcdServiceIP }}:12380 \
7979
--initial-cluster-token={{ .ClusterToken }} \
80-
--initial-advertise-peer-urls=http://{{ .BootEtcdServiceIP }}:12380 \
80+
--initial-advertise-peer-urls=https://{{ .BootEtcdServiceIP }}:12380 \
8181
--skip-hash-check=true
8282
env:
8383
- name: ETCDCTL_API
@@ -119,14 +119,25 @@ spec:
119119
command:
120120
- /usr/local/bin/etcd
121121
- --name=boot-etcd
122-
- --listen-client-urls=http://0.0.0.0:12379
123-
- --listen-peer-urls=http://0.0.0.0:12380
124-
- --advertise-client-urls=http://{{ .BootEtcdServiceIP }}:12379
122+
- --listen-client-urls=https://0.0.0.0:12379
123+
- --listen-peer-urls=https://0.0.0.0:12380
124+
- --advertise-client-urls=https://{{ .BootEtcdServiceIP }}:12379
125125
- --data-dir=/var/etcd/data
126+
- --peer-client-cert-auth=true
127+
- --peer-trusted-ca-file=/etc/kubernetes/secrets/etcdMember/peer-ca-crt.pem
128+
- --peer-cert-file=/etc/kubernetes/secrets/etcdMember/peer-crt.pem
129+
- --peer-key-file=/etc/kubernetes/secrets/etcdMember/peer-key.pem
130+
- --client-cert-auth=true
131+
- --trusted-ca-file=/etc/kubernetes/secrets/etcdMember/client-ca-crt.pem
132+
- --cert-file=/etc/kubernetes/secrets/etcdMember/client-crt.pem
133+
- --key-file=/etc/kubernetes/secrets/etcdMember/client-key.pem
126134
volumeMounts:
127-
- mountPath: /var/etcd
128-
name: etcd
129-
readOnly: false
135+
- mountPath: /var/etcd
136+
name: etcd
137+
readOnly: false
138+
- mountPath: /etc/kubernetes/secrets
139+
name: secrets
140+
readOnly: true
130141
hostNetwork: true
131142
dnsPolicy: ClusterFirstWithHostNet
132143
restartPolicy: Never
@@ -136,6 +147,9 @@ spec:
136147
- name: etcdbackup
137148
hostPath:
138149
path: {{ .BackupDir }}
150+
- name: secrets
151+
hostPath:
152+
path: /etc/kubernetes/bootstrap-secrets
139153
`)
140154

141155
var recoveryEtcdSvcTemplate = []byte(`{

‎pkg/recovery/recover.go‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,8 @@ func (cp *controlPlane) renderBootstrap() (asset.Assets, error) {
119119
if err != nil {
120120
return nil, err
121121
}
122-
requiredConfigMaps, requiredSecrets := fixUpBootstrapPods(pods, cp.bootEtcd != nil)
122+
isSelfHostedEtcd := cp.bootEtcd != nil
123+
requiredConfigMaps, requiredSecrets := fixUpBootstrapPods(pods, isSelfHostedEtcd)
123124
as, err := outputBootstrapPods(pods)
124125
if err != nil {
125126
return nil, err
@@ -129,6 +130,12 @@ func (cp *controlPlane) renderBootstrap() (asset.Assets, error) {
129130
return nil, err
130131
}
131132
as = append(as, configMaps...)
133+
134+
if isSelfHostedEtcd {
135+
requiredSecrets[asset.SecretEtcdMemberPeer] = filepath.Dir(asset.AssetPathSelfHostedEtcdMemberPeerCA)
136+
requiredSecrets[asset.SecretEtcdMemberCli] = filepath.Dir(asset.AssetPathSelfHostedEtcdMemberClientCA)
137+
requiredSecrets[asset.SecretEtcdOperator] = filepath.Dir(asset.AssetPathSelfHostedOperatorEtcdCA)
138+
}
132139
secrets, err := outputBootstrapSecrets(cp.secrets, requiredSecrets)
133140
if err != nil {
134141
return nil, err

‎pkg/util/etcdutil/migrate.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,7 @@ func cleanupBootstrapEtcdService(kubecli kubernetes.Interface) {
237237
}
238238

239239
func detectEtcdTLS(assetDir string) (bool, error) {
240-
etcdCAAssetPath := filepath.Join(assetDir, asset.AssetPathEtcdCA)
240+
etcdCAAssetPath := filepath.Join(assetDir, asset.AssetPathSelfHostedOperatorEtcdCA)
241241
_, err := os.Stat(etcdCAAssetPath)
242242
if err == nil {
243243
return true, nil

‎pkg/util/etcdutil/util.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,9 @@ func WaitClusterReady(endpoint string, etcdTLS *tls.Config) error {
4040

4141
func makeTLSConfig(assetDir string) (*tls.Config, error) {
4242
tlsInfo := transport.TLSInfo{
43-
TrustedCAFile: filepath.Join(assetDir, asset.AssetPathEtcdCA),
44-
CertFile: filepath.Join(assetDir, asset.AssetPathEtcdClientCert),
45-
KeyFile: filepath.Join(assetDir, asset.AssetPathEtcdClientKey),
43+
TrustedCAFile: filepath.Join(assetDir, asset.AssetPathSelfHostedOperatorEtcdCA),
44+
CertFile: filepath.Join(assetDir, asset.AssetPathSelfHostedOperatorEtcdCert),
45+
KeyFile: filepath.Join(assetDir, asset.AssetPathSelfHostedOperatorEtcdKey),
4646
}
4747
return tlsInfo.ClientConfig()
4848
}

0 commit comments

Comments
 (0)