Skip to content
This repository was archived by the owner on Jul 30, 2021. It is now read-only.

Commit 3523114

Browse files
diegsdiegs
authored
Merge pull request #425 from diegs/static-manifests
Use static assets for temporary control plane.
2 parents a135763 + 61d3a72 commit 3523114

File tree

13 files changed

+310
-385
lines changed

13 files changed

+310
-385
lines changed

‎cmd/bootkube/start.go‎

Lines changed: 7 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,11 @@ package main
33
import (
44
"errors"
55
"flag"
6-
"fmt"
7-
"net/url"
86

97
"github.com/spf13/cobra"
108

119
"github.com/kubernetes-incubator/bootkube/pkg/bootkube"
1210
"github.com/kubernetes-incubator/bootkube/pkg/util"
13-
"github.com/kubernetes-incubator/bootkube/pkg/util/etcdutil"
1411
)
1512

1613
var (
@@ -24,36 +21,21 @@ var (
2421
}
2522

2623
startOpts struct {
27-
assetDir string
28-
etcdServer string
29-
selfHostedEtcd bool
24+
assetDir string
25+
podManifestPath string
3026
}
3127
)
3228

3329
func init() {
3430
cmdRoot.AddCommand(cmdStart)
35-
cmdStart.Flags().StringVar(&startOpts.etcdServer, "etcd-server", "http://127.0.0.1:2379", "Single etcd node to use during bootkube bootstrap process.")
3631
cmdStart.Flags().StringVar(&startOpts.assetDir, "asset-dir", "", "Path to the cluster asset directory. Expected layout genereted by the `bootkube render` command.")
37-
cmdStart.Flags().BoolVar(&startOpts.selfHostedEtcd, "experimental-self-hosted-etcd", false, "Self hosted etcd mode. Includes starting the initial etcd member by bootkube.")
32+
cmdStart.Flags().StringVar(&startOpts.podManifestPath, "pod-manifest-path", "/etc/kubernetes/manifests", "The location where the kubelet is configured to look for static pod manifests.")
3833
}
3934

4035
func runCmdStart(cmd *cobra.Command, args []string) error {
41-
etcdServer, err := url.Parse(startOpts.etcdServer)
42-
if err != nil {
43-
return fmt.Errorf("Invalid etcd etcdServer %q: %v", startOpts.etcdServer, err)
44-
}
45-
46-
// TODO: this should likely move into bootkube.Run() eventually.
47-
if startOpts.selfHostedEtcd {
48-
if err := etcdutil.StartEtcd(startOpts.etcdServer); err != nil {
49-
return fmt.Errorf("fail to start etcd: %v", err)
50-
}
51-
}
52-
5336
bk, err := bootkube.NewBootkube(bootkube.Config{
54-
AssetDir: startOpts.assetDir,
55-
EtcdServer: etcdServer,
56-
SelfHostedEtcd: startOpts.selfHostedEtcd,
37+
AssetDir: startOpts.assetDir,
38+
PodManifestPath: startOpts.podManifestPath,
5739
})
5840

5941
if err != nil {
@@ -69,8 +51,8 @@ func runCmdStart(cmd *cobra.Command, args []string) error {
6951
}
7052

7153
func validateStartOpts(cmd *cobra.Command, args []string) error {
72-
if startOpts.etcdServer == "" {
73-
return errors.New("missing required flag: --etcd-server")
54+
if startOpts.podManifestPath == "" {
55+
return errors.New("missing required flag: --pod-manifest-path")
7456
}
7557
if startOpts.assetDir == "" {
7658
return errors.New("missing required flag: --asset-dir")

‎hack/multi-node/bootkube-up‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,9 @@ fi
1212
SELF_HOST_ETCD=${SELF_HOST_ETCD:-false}
1313
if [ ${SELF_HOST_ETCD} = "true" ]; then
1414
echo "WARNING: THIS IS NOT YET FULLY WORKING - merely here to make ongoing testing easier"
15-
etcd_render_flags="--etcd-servers=http://10.3.0.15:2379 --experimental-self-hosted-etcd"
16-
etcd_start_flags="--etcd-server=http://172.17.4.101:12379 --experimental-self-hosted-etcd"
15+
etcd_render_flags="--experimental-self-hosted-etcd"
1716
else
1817
etcd_render_flags="--etcd-servers=http://172.17.4.51:2379"
19-
etcd_start_flags="--etcd-server=http://172.17.4.51:2379"
2018
fi
2119

2220
# Render assets
@@ -36,7 +34,7 @@ scp -q -F ssh_config -r cluster core@c1:/home/core/cluster
3634
scp -q -F ssh_config ../../_output/bin/linux/bootkube core@c1:/home/core
3735

3836
# Run bootkube
39-
ssh -q -F ssh_config core@c1 "sudo GLOG_v=${GLOG_v} /home/core/bootkube start --asset-dir=/home/core/cluster ${etcd_start_flags} 2>> /home/core/bootkube.log"
37+
ssh -q -F ssh_config core@c1 "sudo GLOG_v=${GLOG_v} /home/core/bootkube start --asset-dir=/home/core/cluster 2>> /home/core/bootkube.log"
4038

4139
echo
4240
echo "Bootstrap complete. Access your kubernetes cluster using:"

‎hack/quickstart/init-master.sh‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,13 +36,11 @@ function init_master_node() {
3636
systemctl stop update-engine; systemctl mask update-engine
3737

3838
etcd_render_flags=""
39-
etcd_start_flags=""
4039

4140
# Start etcd.
4241
if [ "$SELF_HOST_ETCD" = true ] ; then
4342
echo "WARNING: THIS IS NOT YET FULLY WORKING - merely here to make ongoing testing easier"
44-
etcd_render_flags="--etcd-servers=http://10.3.0.15:2379 --experimental-self-hosted-etcd"
45-
etcd_start_flags="--etcd-server=http://${COREOS_PRIVATE_IPV4}:12379 --experimental-self-hosted-etcd"
43+
etcd_render_flags="--experimental-self-hosted-etcd"
4644
else
4745
configure_etcd
4846
systemctl enable etcd-member; sudo systemctl start etcd-member
@@ -61,7 +59,7 @@ function init_master_node() {
6159
systemctl enable kubelet; sudo systemctl start kubelet
6260

6361
# Start bootkube to launch a self-hosted cluster
64-
/home/core/bootkube start --asset-dir=/home/core/assets ${etcd_start_flags}
62+
/home/core/bootkube start --asset-dir=/home/core/assets
6563
}
6664

6765
[ "$#" == 1 ] || usage

‎hack/single-node/bootkube-up‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,8 @@ SELF_HOST_ETCD=${SELF_HOST_ETCD:-false}
1313
if [ ${SELF_HOST_ETCD} = "true" ]; then
1414
echo "WARNING: THIS IS NOT YET FULLY WORKING - merely here to make ongoing testing easier"
1515
etcd_render_flags="--experimental-self-hosted-etcd"
16-
etcd_start_flags="--etcd-server=http://172.17.4.100:12379 --experimental-self-hosted-etcd"
1716
else
1817
etcd_render_flags=""
19-
etcd_start_flags=""
2018
fi
2119

2220
# Render assets
@@ -37,7 +35,7 @@ scp -q -F ssh_config -r cluster core@default:/home/core/cluster
3735
scp -q -F ssh_config ../../_output/bin/linux/bootkube core@default:/home/core
3836

3937
# Run bootkube
40-
ssh -q -F ssh_config core@default "sudo GLOG_v=${GLOG_v} /home/core/bootkube start --asset-dir=/home/core/cluster ${etcd_start_flags} 2>> /home/core/bootkube.log"
38+
ssh -q -F ssh_config core@default "sudo GLOG_v=${GLOG_v} /home/core/bootkube start --asset-dir=/home/core/cluster 2>> /home/core/bootkube.log"
4139

4240
echo
4341
echo "Bootstrap complete. Access your kubernetes cluster using:"

‎pkg/asset/asset.go‎

Lines changed: 23 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
)
1515

1616
const (
17+
AssetPathSecrets = "tls"
1718
AssetPathCAKey = "tls/ca.key"
1819
AssetPathCACert = "tls/ca.crt"
1920
AssetPathAPIServerKey = "tls/apiserver.key"
@@ -43,30 +44,39 @@ const (
4344
AssetPathEtcdSvc = "manifests/etcd-service.yaml"
4445
AssetPathKenc = "manifests/kenc.yaml"
4546
AssetPathKubeSystemSARoleBinding = "manifests/kube-system-rbac-role-binding.yaml"
47+
AssetPathBootstrapManifests = "bootstrap-manifests"
48+
AssetPathBootstrapAPIServer = "bootstrap-manifests/bootstrap-apiserver.yaml"
49+
AssetPathBootstrapControllerManager = "bootstrap-manifests/bootstrap-controller-manager.yaml"
50+
AssetPathBootstrapScheduler = "bootstrap-manifests/bootstrap-scheduler.yaml"
51+
AssetPathBootstrapEtcd = "bootstrap-manifests/bootstrap-etcd.yaml"
52+
BootstrapSecretsDir = "/tmp/bootkube/secrets"
4653
)
4754

4855
// AssetConfig holds all configuration needed when generating
4956
// the default set of assets.
5057
type Config struct {
51-
EtcdServers []*url.URL
52-
APIServers []*url.URL
53-
CACert *x509.Certificate
54-
CAPrivKey *rsa.PrivateKey
55-
AltNames *tlsutil.AltNames
56-
PodCIDR *net.IPNet
57-
ServiceCIDR *net.IPNet
58-
APIServiceIP net.IP
59-
DNSServiceIP net.IP
60-
ETCDServiceIP net.IP
61-
SelfHostKubelet bool
62-
SelfHostedEtcd bool
63-
CloudProvider string
58+
EtcdServers []*url.URL
59+
APIServers []*url.URL
60+
CACert *x509.Certificate
61+
CAPrivKey *rsa.PrivateKey
62+
AltNames *tlsutil.AltNames
63+
PodCIDR *net.IPNet
64+
ServiceCIDR *net.IPNet
65+
APIServiceIP net.IP
66+
DNSServiceIP net.IP
67+
ETCDServiceIP net.IP
68+
SelfHostKubelet bool
69+
SelfHostedEtcd bool
70+
CloudProvider string
71+
BootstrapSecretsDir string
6472
}
6573

6674
// NewDefaultAssets returns a list of default assets, optionally
6775
// configured via a user provided AssetConfig. Default assets include
6876
// TLS assets (certs, keys and secrets), and k8s component manifests.
6977
func NewDefaultAssets(conf Config) (Assets, error) {
78+
conf.BootstrapSecretsDir = BootstrapSecretsDir
79+
7080
as := newStaticAssets()
7181
as = append(as, newDynamicAssets(conf)...)
7282

‎pkg/asset/internal/templates.go‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,61 @@ spec:
205205
path: /var/lock
206206
`)
207207

208+
BootstrapAPIServerTemplate = []byte(`apiVersion: v1
209+
kind: Pod
210+
metadata:
211+
name: bootstrap-kube-apiserver
212+
namespace: kube-system
213+
spec:
214+
hostNetwork: true
215+
containers:
216+
- name: kube-apiserver
217+
image: quay.io/coreos/hyperkube:v1.6.1_coreos.0
218+
command:
219+
- /usr/bin/flock
220+
- --exclusive
221+
- --timeout=30
222+
- /var/lock/api-server.lock
223+
- /hyperkube
224+
- apiserver
225+
- --admission-control=NamespaceLifecycle,ServiceAccount
226+
- --allow-privileged=true
227+
- --authorization-mode=RBAC
228+
- --bind-address=0.0.0.0
229+
- --client-ca-file=/etc/kubernetes/secrets/ca.crt
230+
- --etcd-servers={{ range $i, $e := .EtcdServers }}{{ if $i }},{{end}}{{ $e }}{{end}}{{ if .SelfHostedEtcd }},http://127.0.0.1:12379{{end}}
231+
- --insecure-port=8080
232+
- --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt
233+
- --kubelet-client-key=/etc/kubernetes/secrets/apiserver.key
234+
- --runtime-config=api/all=true
235+
- --secure-port=443
236+
- --service-account-key-file=/etc/kubernetes/secrets/service-account.pub
237+
- --service-cluster-ip-range={{ .ServiceCIDR }}
238+
- --storage-backend=etcd3
239+
- --tls-cert-file=/etc/kubernetes/secrets/apiserver.crt
240+
- --tls-private-key-file=/etc/kubernetes/secrets/apiserver.key
241+
volumeMounts:
242+
- mountPath: /etc/ssl/certs
243+
name: ssl-certs-host
244+
readOnly: true
245+
- mountPath: /etc/kubernetes/secrets
246+
name: secrets
247+
readOnly: true
248+
- mountPath: /var/lock
249+
name: var-lock
250+
readOnly: false
251+
volumes:
252+
- name: secrets
253+
hostPath:
254+
path: {{ .BootstrapSecretsDir }}
255+
- name: ssl-certs-host
256+
hostPath:
257+
path: /usr/share/ca-certificates
258+
- name: var-lock
259+
hostPath:
260+
path: /var/lock
261+
`)
262+
208263
KencTemplate = []byte(`apiVersion: "extensions/v1beta1"
209264
kind: DaemonSet
210265
metadata:
@@ -376,6 +431,42 @@ spec:
376431
path: /usr/share/ca-certificates
377432
dnsPolicy: Default # Don't use cluster DNS.
378433
`)
434+
435+
BootstrapControllerManagerTemplate = []byte(`apiVersion: v1
436+
kind: Pod
437+
metadata:
438+
name: bootstrap-kube-controller-manager
439+
namespace: kube-system
440+
spec:
441+
hostNetwork: true
442+
containers:
443+
- name: kube-controller-manager
444+
image: quay.io/coreos/hyperkube:v1.6.1_coreos.0
445+
command:
446+
- ./hyperkube
447+
- controller-manager
448+
- --allocate-node-cidrs=true
449+
- --cluster-cidr={{ .PodCIDR }}
450+
- --configure-cloud-routes=false
451+
- --leader-elect=true
452+
- --master=http://127.0.0.1:8080
453+
- --root-ca-file=/etc/kubernetes/secrets/ca.crt
454+
- --service-account-private-key-file=/etc/kubernetes/secrets/service-account.key
455+
volumeMounts:
456+
- name: secrets
457+
mountPath: /etc/kubernetes/secrets
458+
readOnly: true
459+
- name: ssl-host
460+
mountPath: /etc/ssl/certs
461+
readOnly: true
462+
volumes:
463+
- name: secrets
464+
hostPath:
465+
path: {{ .BootstrapSecretsDir }}
466+
- name: ssl-host
467+
hostPath:
468+
path: /usr/share/ca-certificates
469+
`)
379470
ControllerManagerDisruptionTemplate = []byte(`apiVersion: policy/v1beta1
380471
kind: PodDisruptionBudget
381472
metadata:
@@ -436,6 +527,23 @@ spec:
436527
initialDelaySeconds: 15
437528
timeoutSeconds: 15
438529
530+
`)
531+
532+
BootstrapSchedulerTemplate = []byte(`apiVersion: v1
533+
kind: Pod
534+
metadata:
535+
name: bootstrap-kube-scheduler
536+
namespace: kube-system
537+
spec:
538+
hostNetwork: true
539+
containers:
540+
- name: kube-scheduler
541+
image: quay.io/coreos/hyperkube:v1.6.1_coreos.0
542+
command:
543+
- ./hyperkube
544+
- scheduler
545+
- --leader-elect=true
546+
- --master=http://127.0.0.1:8080
439547
`)
440548
SchedulerDisruptionTemplate = []byte(`apiVersion: policy/v1beta1
441549
kind: PodDisruptionBudget
@@ -716,6 +824,37 @@ spec:
716824
protocol: TCP
717825
`)
718826

827+
BootstrapEtcdTemplate = []byte(`apiVersion: v1
828+
kind: Pod
829+
metadata:
830+
name: bootstrap-etcd
831+
namespace: kube-system
832+
labels:
833+
k8s-app: boot-etcd
834+
spec:
835+
hostNetwork: true
836+
restartPolicy: Never
837+
containers:
838+
- name: etcd
839+
image: quay.io/coreos/etcd:v3.1.0
840+
command:
841+
- /usr/local/bin/etcd
842+
- --name=boot-etcd
843+
- --listen-client-urls=http://0.0.0.0:12379
844+
- --listen-peer-urls=http://0.0.0.0:12380
845+
- --advertise-client-urls=http://$(MY_POD_IP):12379
846+
- --initial-advertise-peer-urls=http://$(MY_POD_IP):12380
847+
- --initial-cluster=boot-etcd=http://$(MY_POD_IP):12380
848+
- --initial-cluster-token=bootkube
849+
- --initial-cluster-state=new
850+
- --data-dir=/var/etcd/data
851+
env:
852+
- name: MY_POD_IP
853+
valueFrom:
854+
fieldRef:
855+
fieldPath: status.podIP
856+
`)
857+
719858
KubeFlannelCfgTemplate = []byte(`apiVersion: v1
720859
kind: ConfigMap
721860
metadata:

‎pkg/asset/k8s.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,9 @@ func newDynamicAssets(conf Config) Assets {
3838
mustCreateAssetFromTemplate(AssetPathProxy, internal.ProxyTemplate, conf),
3939
mustCreateAssetFromTemplate(AssetPathKubeFlannelCfg, internal.KubeFlannelCfgTemplate, conf),
4040
mustCreateAssetFromTemplate(AssetPathKubeDNSSvc, internal.DNSSvcTemplate, conf),
41+
mustCreateAssetFromTemplate(AssetPathBootstrapAPIServer, internal.BootstrapAPIServerTemplate, conf),
42+
mustCreateAssetFromTemplate(AssetPathBootstrapControllerManager, internal.BootstrapControllerManagerTemplate, conf),
43+
mustCreateAssetFromTemplate(AssetPathBootstrapScheduler, internal.BootstrapSchedulerTemplate, conf),
4144
}
4245
if conf.SelfHostKubelet {
4346
assets = append(assets, mustCreateAssetFromTemplate(AssetPathKubelet, internal.KubeletTemplate, conf))
@@ -47,6 +50,7 @@ func newDynamicAssets(conf Config) Assets {
4750
mustCreateAssetFromTemplate(AssetPathEtcdOperator, internal.EtcdOperatorTemplate, conf),
4851
mustCreateAssetFromTemplate(AssetPathEtcdSvc, internal.EtcdSvcTemplate, conf),
4952
mustCreateAssetFromTemplate(AssetPathKenc, internal.KencTemplate, conf),
53+
mustCreateAssetFromTemplate(AssetPathBootstrapEtcd, internal.BootstrapEtcdTemplate, conf),
5054
)
5155
}
5256
return assets

0 commit comments

Comments
 (0)