Skip to content

Commit 0bbbb02

Browse files
Merge pull request #8151 from patrickdillon/gcp-capi-int-lb
CORS-3260: CAPI: Create GCP Internal LB
2 parents 7078bae + 4765e43 commit 0bbbb02

File tree

7 files changed

+262
-71
lines changed

7 files changed

+262
-71
lines changed

pkg/asset/machines/master.go

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,26 @@ func (m *Master) Generate(dependencies asset.Parents) error {
265265
if err != nil {
266266
return err
267267
}
268+
269+
// CAPG-based installs will use only backend services--no target pools,
270+
// so we don't want to include target pools in the control plane machineset.
271+
// TODO(padillon): once this feature gate is the default and we are
272+
// no longer using Terraform, we can update ConfigMasters not to populate this.
273+
if ic.EnabledFeatureGates().Enabled(configv1.FeatureGateClusterAPIInstall) {
274+
for _, machine := range machines {
275+
providerSpec, ok := machine.Spec.ProviderSpec.Value.Object.(*machinev1beta1.GCPMachineProviderSpec)
276+
if !ok {
277+
return errors.New("unable to convert ProviderSpec to GCPMachineProviderSpec")
278+
}
279+
providerSpec.TargetPools = nil
280+
}
281+
cpms := controlPlaneMachineSet.Spec.Template.OpenShiftMachineV1Beta1Machine.Spec.ProviderSpec.Value.Object
282+
providerSpec, ok := cpms.(*machinev1beta1.GCPMachineProviderSpec)
283+
if !ok {
284+
return errors.New("Unable to set target pools to control plane machine set")
285+
}
286+
providerSpec.TargetPools = nil
287+
}
268288
case ibmcloudtypes.Name:
269289
subnets := map[string]string{}
270290
if len(ic.Platform.IBMCloud.ControlPlaneSubnets) > 0 {

pkg/asset/manifests/clusterapi/cluster.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
corev1 "k8s.io/api/core/v1"
1010
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1111
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
12+
"k8s.io/utils/ptr"
1213
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
1314
"sigs.k8s.io/controller-runtime/pkg/client"
1415
"sigs.k8s.io/yaml"
@@ -90,7 +91,11 @@ func (c *Cluster) Generate(dependencies asset.Parents) error {
9091
Name: clusterID.InfraID,
9192
Namespace: capiutils.Namespace,
9293
},
93-
Spec: clusterv1.ClusterSpec{},
94+
Spec: clusterv1.ClusterSpec{
95+
ClusterNetwork: &clusterv1.ClusterNetwork{
96+
APIServerPort: ptr.To[int32](6443),
97+
},
98+
},
9499
}
95100
cluster.SetGroupVersionKind(clusterv1.GroupVersion.WithKind("Cluster"))
96101
c.FileList = append(c.FileList, &asset.RuntimeFile{Object: cluster, File: asset.File{Filename: "01_capi-cluster.yaml"}})

pkg/asset/manifests/gcp/cluster.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"k8s.io/apimachinery/pkg/util/sets"
1111
"k8s.io/utils/ptr"
1212
capg "sigs.k8s.io/cluster-api-provider-gcp/api/v1beta1"
13+
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
1314

1415
"github.com/openshift/installer/pkg/asset"
1516
"github.com/openshift/installer/pkg/asset/installconfig"
@@ -105,8 +106,9 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID
105106
Namespace: capiutils.Namespace,
106107
},
107108
Spec: capg.GCPClusterSpec{
108-
Project: installConfig.Config.GCP.ProjectID,
109-
Region: installConfig.Config.GCP.Region,
109+
Project: installConfig.Config.GCP.ProjectID,
110+
Region: installConfig.Config.GCP.Region,
111+
ControlPlaneEndpoint: clusterv1.APIEndpoint{Port: 6443},
110112
Network: capg.NetworkSpec{
111113
// TODO: Need a network project for installs where the network resources will exist in another
112114
// project such as shared vpc installs

pkg/destroy/gcp/cloudcontroller.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ func (o *ClusterUninstaller) discoverCloudControllerLoadBalancerResources(ctx co
121121
o.insertPendingItems("forwardingrule", found)
122122

123123
// Discover associated health checks: loadBalancerName
124-
found, err = o.listHealthChecksWithFilter(ctx, "items(name),nextPageToken", loadBalancerNameFilter, nil)
124+
found, err = o.listHealthChecksWithFilter(ctx, "healthcheck", "items(name),nextPageToken", loadBalancerNameFilter, o.healthCheckList)
125125
if err != nil {
126126
return err
127127
}
@@ -196,7 +196,7 @@ func (o *ClusterUninstaller) discoverCloudControllerResources(ctx context.Contex
196196
if len(o.cloudControllerUID) > 0 {
197197
// Discover Cloud Controller health checks: k8s-cloudControllerUID-node
198198
filter := fmt.Sprintf("name eq \"k8s-%s-node\"", o.cloudControllerUID)
199-
found, err := o.listHealthChecksWithFilter(ctx, "items(name),nextPageToken", filter, nil)
199+
found, err := o.listHealthChecksWithFilter(ctx, "healthcheck", "items(name),nextPageToken", filter, o.healthCheckList)
200200
if err != nil {
201201
return err
202202
}

pkg/destroy/gcp/healthcheck.go

Lines changed: 72 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package gcp
22

33
import (
44
"context"
5+
"fmt"
56

67
"github.com/pkg/errors"
78
"google.golang.org/api/compute/v1"
@@ -10,54 +11,47 @@ import (
1011
"github.com/openshift/installer/pkg/types/gcp"
1112
)
1213

13-
func (o *ClusterUninstaller) listHealthChecks(ctx context.Context) ([]cloudResource, error) {
14-
return o.listHealthChecksWithFilter(ctx, "items(name),nextPageToken", o.clusterIDFilter(), nil)
14+
func (o *ClusterUninstaller) listHealthChecks(ctx context.Context, typeName string, listFunc healthCheckListFunc) ([]cloudResource, error) {
15+
return o.listHealthChecksWithFilter(ctx, typeName, "items(name),nextPageToken", o.clusterIDFilter(), listFunc)
1516
}
1617

1718
// listHealthChecksWithFilter lists health checks in the project that satisfy the filter criteria.
1819
// The fields parameter specifies which fields should be returned in the result, the filter string contains
1920
// a filter string passed to the API to filter results. The filterFunc is a client-side filtering function
2021
// that determines whether a particular result should be returned or not.
21-
func (o *ClusterUninstaller) listHealthChecksWithFilter(ctx context.Context, fields string, filter string, filterFunc func(*compute.HealthCheck) bool) ([]cloudResource, error) {
22+
func (o *ClusterUninstaller) listHealthChecksWithFilter(ctx context.Context, typeName, fields, filter string, listFunc healthCheckListFunc) ([]cloudResource, error) {
2223
o.Logger.Debugf("Listing health checks")
2324
ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
2425
defer cancel()
2526
result := []cloudResource{}
26-
req := o.computeSvc.HealthChecks.List(o.ProjectID).Fields(googleapi.Field(fields))
27-
if len(filter) > 0 {
28-
req = req.Filter(filter)
29-
}
30-
err := req.Pages(ctx, func(list *compute.HealthCheckList) error {
31-
for _, item := range list.Items {
32-
if filterFunc == nil || filterFunc != nil && filterFunc(item) {
33-
o.Logger.Debugf("Found health check: %s", item.Name)
34-
result = append(result, cloudResource{
35-
key: item.Name,
36-
name: item.Name,
37-
typeName: "healthcheck",
38-
quota: []gcp.QuotaUsage{{
39-
Metric: &gcp.Metric{
40-
Service: gcp.ServiceComputeEngineAPI,
41-
Limit: "health_checks",
42-
},
43-
Amount: 1,
44-
}},
45-
})
46-
}
47-
}
48-
return nil
49-
})
27+
list, err := listFunc(ctx, filter, fields)
5028
if err != nil {
51-
return nil, errors.Wrapf(err, "failed to list health checks")
29+
return nil, fmt.Errorf("failed to list health checks: %w", err)
30+
}
31+
32+
for _, item := range list.Items {
33+
o.Logger.Debugf("Found health check: %s", item.Name)
34+
result = append(result, cloudResource{
35+
key: item.Name,
36+
name: item.Name,
37+
typeName: typeName,
38+
quota: []gcp.QuotaUsage{{
39+
Metric: &gcp.Metric{
40+
Service: gcp.ServiceComputeEngineAPI,
41+
Limit: "health_checks",
42+
},
43+
Amount: 1,
44+
}},
45+
})
5246
}
5347
return result, nil
5448
}
5549

56-
func (o *ClusterUninstaller) deleteHealthCheck(ctx context.Context, item cloudResource) error {
50+
func (o *ClusterUninstaller) deleteHealthCheck(ctx context.Context, item cloudResource, deleteFunc healthCheckDestroyFunc) error {
5751
o.Logger.Debugf("Deleting health check %s", item.name)
5852
ctx, cancel := context.WithTimeout(ctx, defaultTimeout)
5953
defer cancel()
60-
op, err := o.computeSvc.HealthChecks.Delete(o.ProjectID, item.name).RequestId(o.requestID(item.typeName, item.name)).Context(ctx).Do()
54+
op, err := deleteFunc(ctx, item)
6155
if err != nil && !isNoOp(err) {
6256
o.resetRequestID(item.typeName, item.name)
6357
return errors.Wrapf(err, "failed to delete health check %s", item.name)
@@ -77,19 +71,56 @@ func (o *ClusterUninstaller) deleteHealthCheck(ctx context.Context, item cloudRe
7771
// destroyHealthChecks removes all health check resources that have a name prefixed
7872
// with the cluster's infra ID.
7973
func (o *ClusterUninstaller) destroyHealthChecks(ctx context.Context) error {
80-
found, err := o.listHealthChecks(ctx)
81-
if err != nil {
82-
return err
83-
}
84-
items := o.insertPendingItems("healthcheck", found)
85-
for _, item := range items {
86-
err := o.deleteHealthCheck(ctx, item)
74+
for _, hcd := range []healthCheckDestroyer{
75+
{
76+
itemTypeName: "healthcheck",
77+
destroyFunc: o.healthCheckDelete,
78+
listFunc: o.healthCheckList,
79+
},
80+
{
81+
itemTypeName: "regionHealthCheck",
82+
destroyFunc: o.regionHealthCheckDelete,
83+
listFunc: o.regionHealthCheckList,
84+
},
85+
} {
86+
found, err := o.listHealthChecks(ctx, hcd.itemTypeName, hcd.listFunc)
8787
if err != nil {
88-
o.errorTracker.suppressWarning(item.key, err, o.Logger)
88+
return err
89+
}
90+
items := o.insertPendingItems(hcd.itemTypeName, found)
91+
for _, item := range items {
92+
err := o.deleteHealthCheck(ctx, item, hcd.destroyFunc)
93+
if err != nil {
94+
o.errorTracker.suppressWarning(item.key, err, o.Logger)
95+
}
96+
}
97+
if items = o.getPendingItems(hcd.itemTypeName); len(items) > 0 {
98+
return errors.Errorf("%d items pending", len(items))
8999
}
90-
}
91-
if items = o.getPendingItems("healthcheck"); len(items) > 0 {
92-
return errors.Errorf("%d items pending", len(items))
93100
}
94101
return nil
95102
}
103+
104+
type healthCheckListFunc func(ctx context.Context, filter, fields string) (*compute.HealthCheckList, error)
105+
type healthCheckDestroyFunc func(ctx context.Context, item cloudResource) (*compute.Operation, error)
106+
type healthCheckDestroyer struct {
107+
itemTypeName string
108+
destroyFunc healthCheckDestroyFunc
109+
listFunc healthCheckListFunc
110+
}
111+
112+
func (o *ClusterUninstaller) healthCheckDelete(ctx context.Context, item cloudResource) (*compute.Operation, error) {
113+
return o.computeSvc.HealthChecks.Delete(o.ProjectID, item.name).RequestId(o.requestID(item.typeName, item.name)).Context(ctx).Do()
114+
}
115+
116+
func (o *ClusterUninstaller) healthCheckList(ctx context.Context, filter, fields string) (*compute.HealthCheckList, error) {
117+
return o.computeSvc.HealthChecks.List(o.ProjectID).Filter(filter).Fields(googleapi.Field(fields)).Context(ctx).Do()
118+
}
119+
120+
func (o *ClusterUninstaller) regionHealthCheckDelete(ctx context.Context, item cloudResource) (*compute.Operation, error) {
121+
return o.computeSvc.RegionHealthChecks.Delete(o.ProjectID, o.Region, item.name).RequestId(o.requestID(item.typeName, item.name)).Context(ctx).Do()
122+
}
123+
124+
func (o *ClusterUninstaller) regionHealthCheckList(ctx context.Context, filter, fields string) (*compute.HealthCheckList, error) {
125+
return o.computeSvc.RegionHealthChecks.List(o.ProjectID, o.Region).Filter(filter).Fields(googleapi.Field(fields)).Context(ctx).Do()
126+
}

pkg/infrastructure/gcp/clusterapi/clusterapi.go

Lines changed: 41 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import (
44
"context"
55
"encoding/json"
66
"fmt"
7+
"path"
8+
"strings"
79
"time"
810

911
"github.com/sirupsen/logrus"
@@ -13,6 +15,7 @@ import (
1315
"github.com/openshift/installer/pkg/asset/cluster/metadata"
1416
"github.com/openshift/installer/pkg/asset/cluster/tfvars"
1517
"github.com/openshift/installer/pkg/asset/ignition/bootstrap/gcp"
18+
icgcp "github.com/openshift/installer/pkg/asset/installconfig/gcp"
1619
"github.com/openshift/installer/pkg/asset/manifests/capiutils"
1720
"github.com/openshift/installer/pkg/infrastructure/clusterapi"
1821
"github.com/openshift/installer/pkg/types"
@@ -141,13 +144,47 @@ func (p Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput)
141144
logrus.Debugf("publish strategy is set to external but api address is empty")
142145
}
143146

147+
client, err := icgcp.NewClient(context.TODO())
148+
if err != nil {
149+
return err
150+
}
151+
152+
networkProjectID := in.InstallConfig.Config.GCP.NetworkProjectID
153+
if networkProjectID == "" {
154+
networkProjectID = in.InstallConfig.Config.GCP.ProjectID
155+
}
156+
157+
networkSelfLink := *gcpCluster.Status.Network.SelfLink
158+
networkName := path.Base(networkSelfLink)
159+
masterSubnetName := gcptypes.DefaultSubnetName(in.InfraID, "master")
160+
if in.InstallConfig.Config.GCP.ControlPlaneSubnet != "" {
161+
masterSubnetName = in.InstallConfig.Config.GCP.ControlPlaneSubnet
162+
}
163+
164+
subnets, err := client.GetSubnetworks(context.TODO(), networkName, networkProjectID, in.InstallConfig.Config.GCP.Region)
165+
if err != nil {
166+
return fmt.Errorf("failed to retrieve subnets: %w", err)
167+
}
168+
169+
var masterSubnetSelflink string
170+
for _, s := range subnets {
171+
if strings.EqualFold(s.Name, masterSubnetName) {
172+
masterSubnetSelflink = s.SelfLink
173+
break
174+
}
175+
}
176+
177+
if masterSubnetSelflink == "" {
178+
return fmt.Errorf("could not find master subnet %s in subnets %v", masterSubnetName, subnets)
179+
}
180+
181+
zones := gcpCluster.Status.FailureDomains.GetIDs()
182+
144183
// Currently, the internal/private load balancer is not created by CAPG. The load balancer will be created
145-
// by the installer for now
184+
// by the installer for now.
146185
// TODO: remove the creation of the LB and health check here when supported by CAPG.
147186
// https://github.com/kubernetes-sigs/cluster-api-provider-gcp/issues/903
148-
// Create the public (optional) and private load balancer static ip addresses
149-
// TODO: Do we then need to setup a subnet for internal load balancing ?
150-
apiIntIPAddress, err := createInternalLBAddress(ctx, in)
187+
apiIntIPAddress, err := createInternalLB(ctx, in, masterSubnetSelflink, networkSelfLink, zones)
151188
if err != nil {
152189
return fmt.Errorf("failed to create internal load balancer address: %w", err)
153190
}

0 commit comments

Comments
 (0)