Skip to content

Commit cbeb25f

Browse files
richardcaserichardcase
committed
feat: add feature flag and enable gke support
This change adds the feature flag and controllers to enable GKE support in CAPG. Signed-off-by: Richard Case <[email protected]>
1 parent 4cdd048 commit cbeb25f

14 files changed

+240
-83
lines changed

‎cloud/scope/clients.go

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,20 @@ package scope
1818

1919
import (
2020
"context"
21+
"fmt"
2122
"time"
2223

24+
computerest "cloud.google.com/go/compute/apiv1"
25+
container "cloud.google.com/go/container/apiv1"
26+
credentials "cloud.google.com/go/iam/credentials/apiv1"
2327
"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud"
28+
"github.com/pkg/errors"
2429
"google.golang.org/api/compute/v1"
30+
"google.golang.org/api/option"
31+
"k8s.io/client-go/pkg/version"
2532
"k8s.io/client-go/util/flowcontrol"
33+
infrav1 "sigs.k8s.io/cluster-api-provider-gcp/api/v1beta1"
34+
"sigs.k8s.io/controller-runtime/pkg/client"
2635
)
2736

2837
// GCPServices contains all the gcp services used by the scopes.
@@ -57,3 +66,75 @@ func newCloud(project string, service GCPServices) cloud.Cloud {
5766
RateLimiter: &GCPRateLimiter{},
5867
})
5968
}
69+
70+
func defaultClientOptions(ctx context.Context, credentialsRef *infrav1.ObjectReference, crClient client.Client) ([]option.ClientOption, error) {
71+
opts := []option.ClientOption{
72+
option.WithUserAgent(fmt.Sprintf("gcp.cluster.x-k8s.io/%s", version.Get())),
73+
}
74+
75+
if credentialsRef != nil {
76+
rawData, err := getCredentialDataFromRef(ctx, credentialsRef, crClient)
77+
if err != nil {
78+
return nil, fmt.Errorf("getting gcp credentials from reference %s: %w", credentialsRef, err)
79+
}
80+
opts = append(opts, option.WithCredentialsJSON(rawData))
81+
}
82+
83+
return opts, nil
84+
}
85+
86+
func newComputeService(ctx context.Context, credentialsRef *infrav1.ObjectReference, crClient client.Client) (*compute.Service, error) {
87+
opts, err := defaultClientOptions(ctx, credentialsRef, crClient)
88+
if err != nil {
89+
return nil, fmt.Errorf("getting default gcp client options: %w", err)
90+
}
91+
92+
computeSvc, err := compute.NewService(ctx, opts...)
93+
if err != nil {
94+
return nil, fmt.Errorf("creating new compute service instance: %w", err)
95+
}
96+
97+
return computeSvc, nil
98+
}
99+
100+
func newClusterManagerClient(ctx context.Context, credentialsRef *infrav1.ObjectReference, crClient client.Client) (*container.ClusterManagerClient, error) {
101+
opts, err := defaultClientOptions(ctx, credentialsRef, crClient)
102+
if err != nil {
103+
return nil, fmt.Errorf("getting default gcp client options: %w", err)
104+
}
105+
106+
managedClusterClient, err := container.NewClusterManagerClient(ctx, opts...)
107+
if err != nil {
108+
return nil, errors.Errorf("failed to create gcp cluster manager client: %v", err)
109+
}
110+
111+
return managedClusterClient, nil
112+
}
113+
114+
func newIamCredentialsClient(ctx context.Context, credentialsRef *infrav1.ObjectReference, crClient client.Client) (*credentials.IamCredentialsClient, error) {
115+
opts, err := defaultClientOptions(ctx, credentialsRef, crClient)
116+
if err != nil {
117+
return nil, fmt.Errorf("getting default gcp client options: %w", err)
118+
}
119+
120+
credentialsClient, err := credentials.NewIamCredentialsClient(ctx, opts...)
121+
if err != nil {
122+
return nil, errors.Errorf("failed to create gcp ciam credentials client: %v", err)
123+
}
124+
125+
return credentialsClient, nil
126+
}
127+
128+
func newInstanceGroupManagerClient(ctx context.Context, credentialsRef *infrav1.ObjectReference, crClient client.Client) (*computerest.InstanceGroupManagersClient, error) {
129+
opts, err := defaultClientOptions(ctx, credentialsRef, crClient)
130+
if err != nil {
131+
return nil, fmt.Errorf("getting default gcp client options: %w", err)
132+
}
133+
134+
instanceGroupManagersClient, err := computerest.NewInstanceGroupManagersRESTClient(ctx, opts...)
135+
if err != nil {
136+
return nil, errors.Errorf("failed to create gcp instance group managers rest client: %v", err)
137+
}
138+
139+
return instanceGroupManagersClient, nil
140+
}

‎cloud/scope/cluster.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ func NewClusterScope(ctx context.Context, params ClusterScopeParams) (*ClusterSc
5151
}
5252

5353
if params.GCPServices.Compute == nil {
54-
computeSvc, err := createComputeService(ctx, params.GCPCluster.Spec.CredentialsRef, params.Client)
54+
computeSvc, err := newComputeService(ctx, params.GCPCluster.Spec.CredentialsRef, params.Client)
5555
if err != nil {
5656
return nil, errors.Errorf("failed to create gcp compute client: %v", err)
5757
}

‎cloud/scope/credentials.go

Lines changed: 27 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -23,17 +23,16 @@ import (
2323
"os"
2424

2525
"github.com/pkg/errors"
26-
"google.golang.org/api/compute/v1"
27-
"google.golang.org/api/option"
2826
corev1 "k8s.io/api/core/v1"
2927
"k8s.io/apimachinery/pkg/types"
3028
infrav1 "sigs.k8s.io/cluster-api-provider-gcp/api/v1beta1"
3129
"sigs.k8s.io/controller-runtime/pkg/client"
3230
)
3331

3432
const (
35-
// ConfigFilePath is the path to GCP credential config.
36-
ConfigFilePath = "/home/.gcp/credentials"
33+
// ConfigFileEnvVar is the name of the environment variable
34+
// that contains the path to the credentials file.
35+
ConfigFileEnvVar = "GOOGLE_APPLICATION_CREDENTIALS"
3736
)
3837

3938
// Credential is a struct to hold GCP credential data.
@@ -44,6 +43,22 @@ type Credential struct {
4443
ClientID string `json:"client_id"`
4544
}
4645

46+
func getCredentials(ctx context.Context, credentialsRef *infrav1.ObjectReference, crClient client.Client) (*Credential, error) {
47+
var credentialData []byte
48+
var err error
49+
50+
if credentialsRef != nil {
51+
credentialData, err = getCredentialDataFromRef(ctx, credentialsRef, crClient)
52+
} else {
53+
credentialData, err = getCredentialDataUsingADC()
54+
}
55+
if err != nil {
56+
return nil, fmt.Errorf("getting credential data: %w", err)
57+
}
58+
59+
return parseCredential(credentialData)
60+
}
61+
4762
func getCredentialDataFromRef(ctx context.Context, credentialsRef *infrav1.ObjectReference, crClient client.Client) ([]byte, error) {
4863
secretRefName := types.NamespacedName{
4964
Name: credentialsRef.Name,
@@ -63,10 +78,15 @@ func getCredentialDataFromRef(ctx context.Context, credentialsRef *infrav1.Objec
6378
return rawData, nil
6479
}
6580

66-
func getCredentialDataFromMount() ([]byte, error) {
67-
byteValue, err := os.ReadFile(ConfigFilePath)
81+
func getCredentialDataUsingADC() ([]byte, error) {
82+
credsPath := os.Getenv(ConfigFileEnvVar)
83+
if credsPath == "" {
84+
return nil, fmt.Errorf("no ADC environment variable found for credentials (expect %s)", ConfigFileEnvVar)
85+
}
86+
87+
byteValue, err := os.ReadFile(credsPath) //nolint:gosec // We need to read a file here
6888
if err != nil {
69-
return nil, fmt.Errorf("error loading credential from file %s: %w", ConfigFilePath, err)
89+
return nil, fmt.Errorf("reading credentials from file %s: %w", credsPath, err)
7090
}
7191
return byteValue, nil
7292
}
@@ -79,21 +99,3 @@ func parseCredential(rawData []byte) (*Credential, error) {
7999
}
80100
return &credential, nil
81101
}
82-
83-
func createComputeService(ctx context.Context, credentialsRef *infrav1.ObjectReference, crClient client.Client) (*compute.Service, error) {
84-
var computeSvc *compute.Service
85-
var err error
86-
if credentialsRef == nil {
87-
computeSvc, err = compute.NewService(ctx)
88-
} else {
89-
var rawData []byte
90-
rawData, err = getCredentialDataFromRef(ctx, credentialsRef, crClient)
91-
if err == nil {
92-
computeSvc, err = compute.NewService(ctx, option.WithCredentialsJSON(rawData))
93-
}
94-
}
95-
if err != nil {
96-
return nil, errors.Errorf("failed to create gcp compute client: %v", err)
97-
}
98-
return computeSvc, nil
99-
}

‎cloud/scope/managedcluster.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ func NewManagedClusterScope(ctx context.Context, params ManagedClusterScopeParam
5252
}
5353

5454
if params.GCPServices.Compute == nil {
55-
computeSvc, err := createComputeService(ctx, params.GCPManagedCluster.Spec.CredentialsRef, params.Client)
55+
computeSvc, err := newComputeService(ctx, params.GCPManagedCluster.Spec.CredentialsRef, params.Client)
5656
if err != nil {
5757
return nil, errors.Errorf("failed to create gcp compute client: %v", err)
5858
}

‎cloud/scope/managedcontrolplane.go

Lines changed: 4 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,6 @@ import (
2222

2323
"sigs.k8s.io/cluster-api-provider-gcp/util/location"
2424

25-
"google.golang.org/api/option"
26-
2725
"sigs.k8s.io/cluster-api/util/conditions"
2826

2927
container "cloud.google.com/go/container/apiv1"
@@ -64,34 +62,21 @@ func NewManagedControlPlaneScope(ctx context.Context, params ManagedControlPlane
6462
return nil, errors.New("failed to generate new scope from nil GCPManagedControlPlane")
6563
}
6664

67-
var credentialData []byte
68-
var credential *Credential
69-
var err error
70-
if params.GCPManagedCluster.Spec.CredentialsRef != nil {
71-
credentialData, err = getCredentialDataFromRef(ctx, params.GCPManagedCluster.Spec.CredentialsRef, params.Client)
72-
} else {
73-
credentialData, err = getCredentialDataFromMount()
74-
}
75-
if err != nil {
76-
return nil, errors.Errorf("failed to get credential data: %v", err)
77-
}
78-
79-
credential, err = parseCredential(credentialData)
65+
credential, err := getCredentials(ctx, params.GCPManagedCluster.Spec.CredentialsRef, params.Client)
8066
if err != nil {
81-
return nil, errors.Errorf("failed to parse credential data: %v", err)
67+
return nil, fmt.Errorf("getting gcp credentials: %w", err)
8268
}
8369

8470
if params.ManagedClusterClient == nil {
85-
var managedClusterClient *container.ClusterManagerClient
86-
managedClusterClient, err = container.NewClusterManagerClient(ctx, option.WithCredentialsJSON(credentialData))
71+
managedClusterClient, err := newClusterManagerClient(ctx, params.GCPManagedCluster.Spec.CredentialsRef, params.Client)
8772
if err != nil {
8873
return nil, errors.Errorf("failed to create gcp managed cluster client: %v", err)
8974
}
9075
params.ManagedClusterClient = managedClusterClient
9176
}
9277
if params.CredentialsClient == nil {
9378
var credentialsClient *credentials.IamCredentialsClient
94-
credentialsClient, err = credentials.NewIamCredentialsClient(ctx, option.WithCredentialsJSON(credentialData))
79+
credentialsClient, err = newIamCredentialsClient(ctx, params.GCPManagedCluster.Spec.CredentialsRef, params.Client)
9580
if err != nil {
9681
return nil, errors.Errorf("failed to create gcp credentials client: %v", err)
9782
}

‎cloud/scope/managedmachinepool.go

Lines changed: 2 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ import (
2222

2323
"sigs.k8s.io/cluster-api-provider-gcp/util/location"
2424

25-
"google.golang.org/api/option"
2625
"sigs.k8s.io/cluster-api/util/conditions"
2726

2827
compute "cloud.google.com/go/compute/apiv1"
@@ -67,28 +66,15 @@ func NewManagedMachinePoolScope(ctx context.Context, params ManagedMachinePoolSc
6766
return nil, errors.New("failed to generate new scope from nil GCPManagedMachinePool")
6867
}
6968

70-
var credentialData []byte
71-
var err error
72-
if params.GCPManagedCluster.Spec.CredentialsRef != nil {
73-
credentialData, err = getCredentialDataFromRef(ctx, params.GCPManagedCluster.Spec.CredentialsRef, params.Client)
74-
} else {
75-
credentialData, err = getCredentialDataFromMount()
76-
}
77-
if err != nil {
78-
return nil, errors.Errorf("failed to get credential data: %v", err)
79-
}
80-
8169
if params.ManagedClusterClient == nil {
82-
var managedClusterClient *container.ClusterManagerClient
83-
managedClusterClient, err = container.NewClusterManagerClient(ctx, option.WithCredentialsJSON(credentialData))
70+
managedClusterClient, err := newClusterManagerClient(ctx, params.GCPManagedCluster.Spec.CredentialsRef, params.Client)
8471
if err != nil {
8572
return nil, errors.Errorf("failed to create gcp managed cluster client: %v", err)
8673
}
8774
params.ManagedClusterClient = managedClusterClient
8875
}
8976
if params.InstanceGroupManagersClient == nil {
90-
var instanceGroupManagersClient *compute.InstanceGroupManagersClient
91-
instanceGroupManagersClient, err = compute.NewInstanceGroupManagersRESTClient(ctx, option.WithCredentialsJSON(credentialData))
77+
instanceGroupManagersClient, err := newInstanceGroupManagerClient(ctx, params.GCPManagedCluster.Spec.CredentialsRef, params.Client)
9278
if err != nil {
9379
return nil, errors.Errorf("failed to create gcp instance group manager client: %v", err)
9480
}

‎config/manager/manager.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ spec:
2121
containers:
2222
- args:
2323
- --leader-elect
24+
- --feature-gates=GKE=${EXP_CAPG_GKE:=false}
2425
- "--metrics-bind-addr=localhost:8080"
2526
image: controller:latest
2627
imagePullPolicy: IfNotPresent

‎exp/api/v1beta1/gcpmanagedcontrolplane_webhook.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ import (
3030

3131
const (
3232
maxClusterNameLength = 40
33-
resourcePrefix = "capg_"
33+
resourcePrefix = "capg-"
3434
)
3535

3636
// log is for logging in this package.
@@ -89,8 +89,8 @@ func (r *GCPManagedControlPlane) ValidateDelete() error {
8989
}
9090

9191
func generateGKEName(resourceName, namespace string, maxLength int) (string, error) {
92-
escapedName := strings.ReplaceAll(resourceName, ".", "_")
93-
gkeName := fmt.Sprintf("%s_%s", namespace, escapedName)
92+
escapedName := strings.ReplaceAll(resourceName, ".", "-")
93+
gkeName := fmt.Sprintf("%s-%s", namespace, escapedName)
9494

9595
if len(gkeName) < maxLength {
9696
return gkeName, nil

‎exp/controllers/gcpmanagedcluster_controller.go

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@ import (
2525
"github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta"
2626
"github.com/pkg/errors"
2727
apierrors "k8s.io/apimachinery/pkg/api/errors"
28-
"k8s.io/apimachinery/pkg/runtime"
2928
"k8s.io/apimachinery/pkg/types"
3029
"k8s.io/klog/v2"
3130
"sigs.k8s.io/cluster-api-provider-gcp/cloud"
@@ -52,7 +51,6 @@ import (
5251
type GCPManagedClusterReconciler struct {
5352
client.Client
5453
WatchFilterValue string
55-
Scheme *runtime.Scheme
5654
ReconcileTimeout time.Duration
5755
}
5856

@@ -106,7 +104,7 @@ func (r *GCPManagedClusterReconciler) Reconcile(ctx context.Context, req ctrl.Re
106104
Namespace: cluster.Spec.ControlPlaneRef.Namespace,
107105
}
108106

109-
log.V(4).Info("getting control plane %s", controlPlaneRef)
107+
log.V(4).Info("getting control plane ", "ref", controlPlaneRef)
110108
if err := r.Get(ctx, controlPlaneRef, controlPlane); err != nil {
111109
if !apierrors.IsNotFound(err) || gcpCluster.DeletionTimestamp.IsZero() {
112110
return ctrl.Result{}, fmt.Errorf("failed to get control plane ref: %w", err)
@@ -212,15 +210,15 @@ func (r *GCPManagedClusterReconciler) reconcile(ctx context.Context, clusterScop
212210
record.Event(clusterScope.GCPManagedCluster, "GCPManagedClusterReconcile", "Ready")
213211

214212
controlPlaneEndpoint := clusterScope.GCPManagedControlPlane.Spec.Endpoint
213+
clusterScope.SetControlPlaneEndpoint(controlPlaneEndpoint)
214+
215215
if controlPlaneEndpoint.IsZero() {
216216
log.Info("GCPManagedControlplane does not have endpoint yet. Reconciling")
217217
record.Event(clusterScope.GCPManagedCluster, "GCPManagedClusterReconcile", "Waiting for control-plane endpoint")
218-
return ctrl.Result{RequeueAfter: 5 * time.Second}, nil
218+
} else {
219+
record.Eventf(clusterScope.GCPManagedCluster, "GCPManagedClusterReconcile", "Got control-plane endpoint - %s", controlPlaneEndpoint.Host)
219220
}
220221

221-
clusterScope.SetControlPlaneEndpoint(controlPlaneEndpoint)
222-
record.Eventf(clusterScope.GCPManagedCluster, "GCPManagedClusterReconcile", "Got control-plane endpoint - %s", controlPlaneEndpoint.Host)
223-
224222
return ctrl.Result{}, nil
225223
}
226224

‎exp/controllers/gcpmanagedcontrolplane_controller.go

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@ import (
4040
"sigs.k8s.io/controller-runtime/pkg/handler"
4141
"sigs.k8s.io/controller-runtime/pkg/source"
4242

43-
"k8s.io/apimachinery/pkg/runtime"
4443
ctrl "sigs.k8s.io/controller-runtime"
4544
"sigs.k8s.io/controller-runtime/pkg/client"
4645
"sigs.k8s.io/controller-runtime/pkg/log"
@@ -50,7 +49,6 @@ import (
5049
type GCPManagedControlPlaneReconciler struct {
5150
client.Client
5251
ReconcileTimeout time.Duration
53-
Scheme *runtime.Scheme
5452
WatchFilterValue string
5553
}
5654

0 commit comments

Comments
 (0)