Skip to content
This repository was archived by the owner on Aug 12, 2025. It is now read-only.

Commit 620918f

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #142 from gianarb/feature/multi-master
Kubernetes HA with multi control plane
2 parents cd4f7d7 + e936655 commit 620918f

File tree

10 files changed

+258
-107
lines changed

10 files changed

+258
-107
lines changed

‎controllers/packetcluster_controller.go

Lines changed: 31 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ package controllers
1818

1919
import (
2020
"context"
21-
"fmt"
2221
"time"
2322

2423
"github.com/go-logr/logr"
@@ -33,6 +32,7 @@ import (
3332
"sigs.k8s.io/controller-runtime/pkg/handler"
3433
"sigs.k8s.io/controller-runtime/pkg/source"
3534

35+
"github.com/packethost/cluster-api-provider-packet/api/v1alpha3"
3636
infrastructurev1alpha3 "github.com/packethost/cluster-api-provider-packet/api/v1alpha3"
3737
packet "github.com/packethost/cluster-api-provider-packet/pkg/cloud/packet"
3838
"github.com/packethost/cluster-api-provider-packet/pkg/cloud/packet/scope"
@@ -102,28 +102,42 @@ func (r *PacketClusterReconciler) Reconcile(req ctrl.Request) (_ ctrl.Result, re
102102
}
103103
}()
104104

105-
clusterScope.PacketCluster.Status.Ready = true
105+
// Handle deleted clusters
106+
if !cluster.DeletionTimestamp.IsZero() {
107+
return r.reconcileDelete(clusterScope)
108+
}
106109

107-
address, err := r.getIP(clusterScope.PacketCluster)
108-
_, isNoMachine := err.(*MachineNotFound)
109-
_, isNoIP := err.(*MachineNoIP)
110-
switch {
111-
case err != nil && isNoMachine:
112-
logger.Info("Control plane device not found. Requeueing...")
113-
return ctrl.Result{Requeue: true, RequeueAfter: 30 * time.Second}, nil
114-
case err != nil && isNoIP:
115-
logger.Info("Control plane device not found. Requeueing...")
116-
return ctrl.Result{Requeue: true, RequeueAfter: 30 * time.Second}, nil
117-
case err != nil:
118-
logger.Error(err, "error getting a control plane ip")
119-
return ctrl.Result{}, err
120-
case err == nil:
110+
return r.reconcileNormal(packetcluster, clusterScope)
111+
}
112+
113+
func (r *PacketClusterReconciler) reconcileNormal(packetcluster *v1alpha3.PacketCluster, clusterScope *scope.ClusterScope) (ctrl.Result, error) {
114+
if ipReserv, err := r.PacketClient.GetIPByClusterIdentifier(clusterScope.Namespace(), clusterScope.Name(), packetcluster.Spec.ProjectID); err == packet.ErrControlPlanEndpointNotFound {
115+
// There is not an ElasticIP with the right tags, at this point we can create one
116+
ip, err := r.PacketClient.CreateIP(clusterScope.Namespace(), clusterScope.Name(), packetcluster.Spec.ProjectID, packetcluster.Spec.Facility)
117+
if err != nil {
118+
r.Log.Error(err, "error reserving an ip")
119+
return ctrl.Result{}, err
120+
}
121121
clusterScope.PacketCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{
122-
Host: address,
122+
Host: ip.To4().String(),
123+
Port: 6443,
124+
}
125+
} else {
126+
// If there is an ElasticIP with the right tag just use it again
127+
clusterScope.PacketCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{
128+
Host: ipReserv.Address,
123129
Port: 6443,
124130
}
125131
}
132+
clusterScope.PacketCluster.Status.Ready = true
133+
return ctrl.Result{}, nil
134+
}
126135

136+
func (r *PacketClusterReconciler) reconcileDelete(clusterScope *scope.ClusterScope) (ctrl.Result, error) {
137+
// Initially I created this handler to remove an elastic IP when a cluster
138+
// gets delete, but it does not sound like a good idea. It is better to
139+
// leave to the users the ability to decide if they want to keep and resign
140+
// the IP or if they do not need it anymore
127141
return ctrl.Result{}, nil
128142
}
129143

@@ -139,30 +153,6 @@ func (r *PacketClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
139153
Complete(r)
140154
}
141155

142-
func (r *PacketClusterReconciler) getIP(cluster *infrastructurev1alpha3.PacketCluster) (string, error) {
143-
if cluster == nil {
144-
return "", fmt.Errorf("cannot get IP of machine in nil cluster")
145-
}
146-
tags := []string{
147-
packet.GenerateClusterTag(string(cluster.Name)),
148-
infrastructurev1alpha3.MasterTag,
149-
}
150-
device, err := r.PacketClient.GetDeviceByTags(cluster.Spec.ProjectID, tags)
151-
if err != nil {
152-
return "", fmt.Errorf("error retrieving machine: %v", err)
153-
}
154-
if device == nil {
155-
return "", &MachineNotFound{err: fmt.Sprintf("machine does not exist")}
156-
}
157-
if device.Network == nil || len(device.Network) == 0 || device.Network[0].Address == "" {
158-
return "", &MachineNoIP{err: "machine does not yet have an IP address"}
159-
}
160-
// TODO: validate that this address exists, so we don't hit nil pointer
161-
// TODO: check which address to return
162-
// TODO: check address format (cidr, subnet, etc.)
163-
return device.Network[0].Address, nil
164-
}
165-
166156
// MachineNotFound error representing that the requested device was not yet found
167157
type MachineNotFound struct {
168158
err string

‎controllers/packetmachine_controller.go

Lines changed: 52 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ import (
2222
"net/http"
2323
"time"
2424

25+
corev1 "k8s.io/api/core/v1"
26+
2527
"github.com/go-logr/logr"
2628
"github.com/google/uuid"
2729
"github.com/packethost/packngo"
@@ -47,6 +49,7 @@ import (
4749

4850
const (
4951
providerName = "packet"
52+
force = true
5053
)
5154

5255
// PacketMachineReconciler reconciles a PacketMachine object
@@ -192,8 +195,10 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
192195

193196
providerID := machineScope.GetInstanceID()
194197
var (
195-
dev *packngo.Device
196-
err error
198+
dev *packngo.Device
199+
addrs []corev1.NodeAddress
200+
err error
201+
controlPlaneEndpoint packngo.IPAddressReservation
197202
)
198203
// if we have no provider ID, then we are creating
199204
if providerID != "" {
@@ -203,17 +208,35 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
203208
}
204209
}
205210
if dev == nil {
206-
// generate a unique UID that will survive pivot, i.e. is not tied to the cluster itself
207211
mUID := uuid.New().String()
208212
tags := []string{
209213
packet.GenerateMachineTag(mUID),
210214
packet.GenerateClusterTag(clusterScope.Name()),
211215
}
212216

213-
name := machineScope.Name()
214-
dev, err = r.PacketClient.NewDevice(name, clusterScope.PacketCluster.Spec.ProjectID, machineScope, tags)
217+
// when the node is a control plan we should check if the elastic ip
218+
// for this cluster is not assigned. If it is free we can prepare the
219+
// current node to use it.
220+
if machineScope.IsControlPlane() {
221+
controlPlaneEndpoint, _ = r.PacketClient.GetIPByClusterIdentifier(
222+
clusterScope.Namespace(),
223+
clusterScope.Name(),
224+
clusterScope.PacketCluster.Spec.ProjectID)
225+
if len(controlPlaneEndpoint.Assignments) == 0 {
226+
a := corev1.NodeAddress{
227+
Type: corev1.NodeExternalIP,
228+
Address: controlPlaneEndpoint.Address,
229+
}
230+
addrs = append(addrs, a)
231+
// This tag is currently not used. I placed it there to easely localize the device that currently has the IP attached.
232+
// Probably it is not neeed but I wil get back to it when working at #141.
233+
tags = append(tags, fmt.Sprintf("capp.control-plane-endpoint=%s", controlPlaneEndpoint.Address))
234+
}
235+
}
236+
237+
dev, err = r.PacketClient.NewDevice(machineScope, tags)
215238
if err != nil {
216-
errs := fmt.Errorf("failed to create machine %s: %v", name, err)
239+
errs := fmt.Errorf("failed to create machine %s: %v", machineScope.Name(), err)
217240
machineScope.SetErrorReason(capierrors.CreateMachineError)
218241
machineScope.SetErrorMessage(errs)
219242
return ctrl.Result{}, errs
@@ -224,12 +247,13 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
224247
machineScope.SetProviderID(dev.ID)
225248
machineScope.SetInstanceStatus(infrastructurev1alpha3.PacketResourceStatus(dev.State))
226249

227-
addrs, err := r.PacketClient.GetDeviceAddresses(dev)
250+
deviceAddr, err := r.PacketClient.GetDeviceAddresses(dev)
228251
if err != nil {
229252
machineScope.SetErrorMessage(errors.New("failed to getting device addresses"))
230253
return ctrl.Result{}, err
231254
}
232-
machineScope.SetAddresses(addrs)
255+
256+
machineScope.SetAddresses(append(addrs, deviceAddr...))
233257

234258
// Proceed to reconcile the PacketMachine state.
235259
var result = ctrl.Result{}
@@ -240,6 +264,25 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
240264
result = ctrl.Result{RequeueAfter: 10 * time.Second}
241265
case infrastructurev1alpha3.PacketResourceStatusRunning:
242266
machineScope.Info("Machine instance is active", "instance-id", machineScope.GetInstanceID())
267+
268+
// This logic is here because an elastic ip can be assigned only an
269+
// active node. It needs to be a control plane and the IP should not be
270+
// assigned to anything at this point.
271+
controlPlaneEndpoint, _ = r.PacketClient.GetIPByClusterIdentifier(
272+
clusterScope.Namespace(),
273+
clusterScope.Name(),
274+
clusterScope.PacketCluster.Spec.ProjectID)
275+
if len(controlPlaneEndpoint.Assignments) == 0 && machineScope.IsControlPlane() {
276+
if _, _, err := r.PacketClient.DeviceIPs.Assign(dev.ID, &packngo.AddressStruct{
277+
Address: controlPlaneEndpoint.Address,
278+
}); err != nil {
279+
r.Log.Error(err, "err assigining elastic ip to control plane. retrying...")
280+
return ctrl.Result{
281+
Requeue: true,
282+
RequeueAfter: time.Second * 20,
283+
}, nil
284+
}
285+
}
243286
machineScope.SetReady()
244287
result = ctrl.Result{}
245288
default:
@@ -279,7 +322,7 @@ func (r *PacketMachineReconciler) reconcileDelete(ctx context.Context, machineSc
279322
return ctrl.Result{}, fmt.Errorf("machine does not exist: %s", packetmachine.Name)
280323
}
281324

282-
_, err = r.PacketClient.Devices.Delete(device.ID)
325+
_, err = r.PacketClient.Devices.Delete(device.ID, force)
283326
if err != nil {
284327
return ctrl.Result{}, fmt.Errorf("failed to delete the machine: %v", err)
285328
}

‎docs/concepts/cluster.md

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
The PacketCluster is the CRD that contains information about where to place the
2+
Kubernetes cluster: facility and project.
3+
4+
We do not support cross facilities or multi projects cluster. If you need so my
5+
suggestion is to look for what it is called [federation](k8s-federation).
6+
7+
## Topology
8+
9+
Each cluster we create leverages at least two Packet features: Device and ElasticIP.
10+
11+
Kubernetes node is a Device and each cluster has an
12+
[ElasticIP](elastic-ip-packet) that is tagged with the name of the cluster.
13+
14+
The ElasticIP guarantees a stable endpoint even when the control plane(s) are
15+
recycling during a Kubernetes version update or an outages.
16+
17+
## ElasticIP lifecycle
18+
19+
Every cluster has its own ElasticIP. It is tagged with the name of the cluster and
20+
it does not get removed when a cluster is terminated. You have to remove it manually.
21+
22+
This is a safety feature in this way you can re-assign the IP to another
23+
cluster with the same name.
24+
25+
## FAQ
26+
27+
**Does cluster-api work with only Ubuntu/Debian?**
28+
29+
Currently the cluster-template only supports Ubuntu because it uses `apt` and it
30+
does a couple of assumptions around networking. This does not mean that you
31+
can't use `cluster-api-provier-packet` with other templates, but you will have
32+
to make your own cluster specification in order to make the installation process
33+
to work as you want. We have an open issue about this: ["Figure out where we
34+
stand about operating systems"](os-issue).
35+
36+
[k8s-federation]: https://kubernetes.io/blog/2018/12/12/kubernetes-federation-evolution/
37+
[elastic-ip-packet]: https://www.packet.com/developers/docs/network/basic/elastic-ips/
38+
[os-issue]: https://github.com/kubernetes-sigs/cluster-api-provider-packet/issues/118

‎docs/concepts/kubeadmcontrolplane.md

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
[KubeadmControlPlane](kubeadmcontrolplane-book) is a CRD provided by the cluster-api kubeadm bootstrapped.
2+
It manages the control planes lifecycle.
3+
4+
We use this CRD when deploying a Kubernetes Cluster in High Availability because
5+
you can use the field `replicas` to specify how many control plans you need.
6+
7+
It is a good idea to use it even when you have only one control plane because it
8+
manages Kubernetes updates.
9+
10+
[kubeadmcontrolplane-book]: https://cluster-api.sigs.k8s.io/developer/architecture/controllers/control-plane.html

‎docs/concepts/machine.md

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,6 @@ metadata:
1010
name: "qa-master-0"
1111
spec:
1212
OS: "ubuntu_18_04"
13-
facility:
14-
- "dfw2"
1513
billingCycle: hourly
1614
machineType: "t2.small"
1715
sshKeys:

‎go.mod

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,5 @@ require (
1717
sigs.k8s.io/cluster-api v0.3.5
1818
sigs.k8s.io/controller-runtime v0.5.2
1919
)
20+
21+
replace github.com/packethost/packngo => github.com/deitch/packngo v0.2.1-0.20200628082620-d644bb21e1f3

‎go.sum

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,8 @@ github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2
7474
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
7575
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
7676
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
77+
github.com/deitch/packngo v0.2.1-0.20200628082620-d644bb21e1f3 h1:ph32bLs6ix+iPhSP4oapLhlpIwW1n9I9e7wELpnvBfI=
78+
github.com/deitch/packngo v0.2.1-0.20200628082620-d644bb21e1f3/go.mod h1:erURcsqYzwc9wSb04TX4so+s6F3uZtbXUil0W1LCGHA=
7779
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
7880
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
7981
github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
@@ -210,6 +212,12 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de
210212
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
211213
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
212214
github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
215+
github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
216+
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
217+
github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
218+
github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
219+
github.com/hashicorp/go-retryablehttp v0.6.6 h1:HJunrbHTDDbBb/ay4kxa1n+dLmttUlnP3V9oNE4hmsM=
220+
github.com/hashicorp/go-retryablehttp v0.6.6/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
213221
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
214222
github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
215223
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -314,8 +322,6 @@ github.com/onsi/gomega v1.9.0 h1:R1uwffexN6Pr340GtYRIdZmAiN4J+iw6WG4wog1DUXg=
314322
github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
315323
github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
316324
github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
317-
github.com/packethost/packngo v0.2.0 h1:mSlzOof8PsOWCy78sBMt/PwMJTEjjQ/rRvMixu4Nm6c=
318-
github.com/packethost/packngo v0.2.0/go.mod h1:RQHg5xR1F614BwJyepfMqrKN+32IH0i7yX+ey43rEeQ=
319325
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
320326
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
321327
github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4=
@@ -397,6 +403,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
397403
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
398404
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
399405
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
406+
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
407+
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
400408
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
401409
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
402410
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
@@ -439,6 +447,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq
439447
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
440448
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM=
441449
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
450+
golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a h1:y6sBfNd1b9Wy08a6K1Z1DZc4aXABUN5TKjkYhz7UKmo=
451+
golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
442452
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
443453
golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
444454
golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=

0 commit comments

Comments
 (0)