Skip to content
This repository was archived by the owner on Jul 30, 2021. It is now read-only.

Commit c961a5d

Browse files
hongchaodenghongchaodeng
committed
self hosted etcd migration
1 parent 4152ac3 commit c961a5d

File tree

9 files changed

+273
-28
lines changed

9 files changed

+273
-28
lines changed

‎cmd/bootkube/render.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ var (
3434
apiServers string
3535
altNames string
3636
selfHostKubelet bool
37+
storageBackend string
3738
}
3839
)
3940

@@ -43,6 +44,7 @@ func init() {
4344
cmdRender.Flags().StringVar(&renderOpts.caCertificatePath, "ca-certificate-path", "", "Path to an existing PEM encoded CA. If provided, TLS assets will be generated using this certificate authority.")
4445
cmdRender.Flags().StringVar(&renderOpts.caPrivateKeyPath, "ca-private-key-path", "", "Path to an existing Certificate Authority RSA private key. Required if --ca-certificate is set.")
4546
cmdRender.Flags().StringVar(&renderOpts.etcdServers, "etcd-servers", "http://127.0.0.1:2379", "List of etcd servers URLs including host:port, comma separated")
47+
cmdRender.Flags().StringVar(&renderOpts.storageBackend, "storage-backend", "etcd2", "storage backend for APIServer. Supports: etcd2, etcd3.")
4648
cmdRender.Flags().StringVar(&renderOpts.apiServers, "api-servers", "https://127.0.0.1:443", "List of API server URLs including host:port, commma seprated")
4749
cmdRender.Flags().StringVar(&renderOpts.altNames, "api-server-alt-names", "", "List of SANs to use in api-server certificate. Example: 'IP=127.0.0.1,IP=127.0.0.2,DNS=localhost'. If empty, SANs will be extracted from the --api-servers flag.")
4850
cmdRender.Flags().BoolVar(&renderOpts.selfHostKubelet, "self-host-kubelet", false, "Create a self-hosted kubelet daemonset.")
@@ -114,6 +116,7 @@ func flagsToAssetConfig() (c *asset.Config, err error) {
114116
APIServers: apiServers,
115117
AltNames: altNames,
116118
SelfHostKubelet: renderOpts.selfHostKubelet,
119+
StorageBackend: renderOpts.storageBackend,
117120
}, nil
118121
}
119122

‎cmd/bootkube/start.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,9 @@ func runCmdStart(cmd *cobra.Command, args []string) error {
5151
}
5252

5353
bk, err := bootkube.NewBootkube(bootkube.Config{
54-
AssetDir: startOpts.assetDir,
55-
EtcdServer: etcdServer,
54+
AssetDir: startOpts.assetDir,
55+
EtcdServer: etcdServer,
56+
SelfHostedEtcd: startOpts.selfHostedEtcd,
5657
})
5758

5859
if err != nil {

‎pkg/asset/asset.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ const (
3434
AssetPathKubeDNSDeployment = "manifests/kube-dns-deployment.yaml"
3535
AssetPathKubeDNSSvc = "manifests/kube-dns-svc.yaml"
3636
AssetPathSystemNamespace = "manifests/kube-system-ns.yaml"
37+
AssetPathEtcdOperator = "manifests/etcd-operator.yaml"
38+
AssetPathEtcdSvc = "manifests/etcd-service.yaml"
3739
)
3840

3941
// AssetConfig holds all configuration needed when generating
@@ -45,6 +47,7 @@ type Config struct {
4547
CAPrivKey *rsa.PrivateKey
4648
AltNames *tlsutil.AltNames
4749
SelfHostKubelet bool
50+
StorageBackend string
4851
}
4952

5053
// NewDefaultAssets returns a list of default assets, optionally

‎pkg/asset/internal/templates.go‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,7 @@ spec:
133133
- --insecure-port=8080
134134
- --advertise-address=$(MY_POD_IP)
135135
- --etcd-servers={{ range $i, $e := .EtcdServers }}{{ if $i }},{{end}}{{ $e }}{{end}}
136+
- --storage-backend={{.StorageBackend}}
136137
- --allow-privileged=true
137138
- --service-cluster-ip-range=10.3.0.0/24
138139
- --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota
@@ -418,5 +419,43 @@ spec:
418419
- name: dns-tcp
419420
port: 53
420421
protocol: TCP
422+
`)
423+
EtcdOperatorTemplate = []byte(`apiVersion: extensions/v1beta1
424+
kind: Deployment
425+
metadata:
426+
name: etcd-operator
427+
namespace: kube-system
428+
labels:
429+
k8s-app: etcd-operator
430+
spec:
431+
replicas: 1
432+
template:
433+
metadata:
434+
labels:
435+
k8s-app: etcd-operator
436+
spec:
437+
containers:
438+
- name: etcd-operator
439+
image: quay.io/coreos/etcd-operator
440+
env:
441+
- name: MY_POD_NAMESPACE
442+
valueFrom:
443+
fieldRef:
444+
fieldPath: metadata.namespace
445+
`)
446+
EtcdSvcTemplate = []byte(`apiVersion: v1
447+
kind: Service
448+
metadata:
449+
name: etcd-service
450+
namespace: kube-system
451+
spec:
452+
selector:
453+
app: etcd
454+
etcd_cluster: etcd-cluster
455+
clusterIP: 10.3.0.15
456+
ports:
457+
- name: client
458+
port: 2379
459+
protocol: TCP
421460
`)
422461
)

‎pkg/asset/k8s.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ func newStaticAssets(selfHostKubelet bool) Assets {
2626
mustCreateAssetFromTemplate(AssetPathKubeDNSDeployment, internal.DNSDeploymentTemplate, noData),
2727
mustCreateAssetFromTemplate(AssetPathKubeDNSSvc, internal.DNSSvcTemplate, noData),
2828
mustCreateAssetFromTemplate(AssetPathCheckpointer, internal.CheckpointerTemplate, noData),
29+
mustCreateAssetFromTemplate(AssetPathEtcdOperator, internal.EtcdOperatorTemplate, noData),
30+
mustCreateAssetFromTemplate(AssetPathEtcdSvc, internal.EtcdSvcTemplate, noData),
2931
}
3032
if selfHostKubelet {
3133
assets = append(assets, mustCreateAssetFromTemplate(AssetPathKubelet, internal.KubeletTemplate, noData))

‎pkg/bootkube/bootkube.go‎

Lines changed: 36 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -30,35 +30,24 @@ var requiredPods = []string{
3030
}
3131

3232
type Config struct {
33-
AssetDir string
34-
EtcdServer *url.URL
33+
AssetDir string
34+
EtcdServer *url.URL
35+
SelfHostedEtcd bool
3536
}
3637

3738
type bootkube struct {
38-
assetDir string
39-
apiServer *apiserver.APIServer
40-
controller *controller.CMServer
41-
scheduler *scheduler.SchedulerServer
39+
selfHostedEtcd bool
40+
assetDir string
41+
apiServer *apiserver.APIServer
42+
controller *controller.CMServer
43+
scheduler *scheduler.SchedulerServer
4244
}
4345

4446
func NewBootkube(config Config) (*bootkube, error) {
4547
apiServer := apiserver.NewAPIServer()
4648
fs := pflag.NewFlagSet("apiserver", pflag.ExitOnError)
4749
apiServer.AddFlags(fs)
48-
fs.Parse([]string{
49-
"--bind-address=0.0.0.0",
50-
"--secure-port=443",
51-
"--insecure-port=8080",
52-
"--allow-privileged=true",
53-
"--tls-private-key-file=" + filepath.Join(config.AssetDir, asset.AssetPathAPIServerKey),
54-
"--tls-cert-file=" + filepath.Join(config.AssetDir, asset.AssetPathAPIServerCert),
55-
"--client-ca-file=" + filepath.Join(config.AssetDir, asset.AssetPathCACert),
56-
"--etcd-servers=" + config.EtcdServer.String(),
57-
"--service-cluster-ip-range=10.3.0.0/24",
58-
"--service-account-key-file=" + filepath.Join(config.AssetDir, asset.AssetPathServiceAccountPubKey),
59-
"--admission-control=NamespaceLifecycle,ServiceAccount",
60-
"--runtime-config=api/all=true",
61-
})
50+
fs.Parse(makeAPIServerFlags(config))
6251

6352
cmServer := controller.NewCMServer()
6453
fs = pflag.NewFlagSet("controllermanager", pflag.ExitOnError)
@@ -79,13 +68,35 @@ func NewBootkube(config Config) (*bootkube, error) {
7968
})
8069

8170
return &bootkube{
82-
apiServer: apiServer,
83-
controller: cmServer,
84-
scheduler: schedServer,
85-
assetDir: config.AssetDir,
71+
apiServer: apiServer,
72+
controller: cmServer,
73+
scheduler: schedServer,
74+
assetDir: config.AssetDir,
75+
selfHostedEtcd: config.SelfHostedEtcd,
8676
}, nil
8777
}
8878

79+
func makeAPIServerFlags(config Config) []string {
80+
res := []string{
81+
"--bind-address=0.0.0.0",
82+
"--secure-port=443",
83+
"--insecure-port=8080",
84+
"--allow-privileged=true",
85+
"--tls-private-key-file=" + filepath.Join(config.AssetDir, asset.AssetPathAPIServerKey),
86+
"--tls-cert-file=" + filepath.Join(config.AssetDir, asset.AssetPathAPIServerCert),
87+
"--client-ca-file=" + filepath.Join(config.AssetDir, asset.AssetPathCACert),
88+
"--etcd-servers=" + config.EtcdServer.String(),
89+
"--service-cluster-ip-range=10.3.0.0/24",
90+
"--service-account-key-file=" + filepath.Join(config.AssetDir, asset.AssetPathServiceAccountPubKey),
91+
"--admission-control=NamespaceLifecycle,ServiceAccount",
92+
"--runtime-config=api/all=true",
93+
}
94+
if config.SelfHostedEtcd {
95+
res = append(res, "--storage-backend=etcd3")
96+
}
97+
return res
98+
}
99+
89100
func (b *bootkube) Run() error {
90101
UserOutput("Running temporary bootstrap control plane...\n")
91102

@@ -98,7 +109,7 @@ func (b *bootkube) Run() error {
98109
errch <- err
99110
}
100111
}()
101-
go func() { errch <- WaitUntilPodsRunning(requiredPods, assetTimeout) }()
112+
go func() { errch <- WaitUntilPodsRunning(requiredPods, assetTimeout, b.selfHostedEtcd) }()
102113

103114
// If any of the bootkube services exit, it means it is unrecoverable and we should exit.
104115
err := <-errch

‎pkg/bootkube/status.go‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"time"
99

1010
"github.com/golang/glog"
11+
"github.com/kubernetes-incubator/bootkube/pkg/util/etcdutil"
1112
"k8s.io/kubernetes/pkg/api"
1213
"k8s.io/kubernetes/pkg/api/v1"
1314
"k8s.io/kubernetes/pkg/client/cache"
@@ -24,7 +25,7 @@ const (
2425
doesNotExist = "DoesNotExist"
2526
)
2627

27-
func WaitUntilPodsRunning(pods []string, timeout time.Duration) error {
28+
func WaitUntilPodsRunning(pods []string, timeout time.Duration, selfHostedEtcd bool) error {
2829
sc, err := NewStatusController(pods)
2930
if err != nil {
3031
return err
@@ -35,6 +36,12 @@ func WaitUntilPodsRunning(pods []string, timeout time.Duration) error {
3536
return fmt.Errorf("error while checking pod status: %v", err)
3637
}
3738

39+
if selfHostedEtcd {
40+
if err := etcdutil.Migrate(); err != nil {
41+
return err
42+
}
43+
}
44+
3845
UserOutput("All self-hosted control plane components successfully started\n")
3946
return nil
4047
}

0 commit comments

Comments
 (0)