Skip to content

Commit 5146ab8

Browse files
authored
feat: add support for clusters in multiple regions (#44)
Adding `ClientProvider` allows clusters to use different `OCIClients` groups to interact with the regional APIs.
1 parent 76da8d3 commit 5146ab8

File tree

24 files changed

+1409
-120
lines changed

24 files changed

+1409
-120
lines changed

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ bin
1111

1212
# Output of the go coverage tool, specifically when used with LiteIDE
1313
*.out
14+
cover.html
1415
testbin/
1516
out/
1617

Makefile

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ ARTIFACTS ?= $(ROOT_DIR)/_artifacts
4848
KUBETEST_CONF_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/conformance.yaml)
4949
KUBETEST_FAST_CONF_PATH ?= $(abspath $(E2E_DATA_DIR)/kubetest/conformance-fast.yaml)
5050
GINKGO_FOCUS ?= Workload cluster creation
51-
GINKGO_SKIP ?= "Bare Metal"
51+
GINKGO_SKIP ?= "Bare Metal|Multi-Region"
5252
# Image URL to use all building/pushing image targets
5353
IMG ?= controller:latest
5454
# Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
@@ -106,7 +106,8 @@ ENVTEST_ASSETS_DIR=$(shell pwd)/testbin
106106
test: manifests generate fmt vet ## Run tests.
107107
mkdir -p ${ENVTEST_ASSETS_DIR}
108108
test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.8.3/hack/setup-envtest.sh
109-
source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./... -coverprofile cover.out
109+
source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./... -coverprofile cover.out && go tool cover -html=cover.out -o cover.html
110+
110111

111112
##@ Build
112113

@@ -241,6 +242,7 @@ serve-book: build-book ## Build and serve the book with live-reloading enabled
241242
.PHONY: generate-e2e-templates ## Generate OCI infrastructure templates for e2e test suite.
242243
generate-e2e-templates: kustomize
243244
$(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template --load_restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template.yaml
245+
$(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-alternative-region --load_restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-alternative-region.yaml
244246
$(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-bare-metal --load_restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-bare-metal.yaml
245247
$(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-md-remediation --load_restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-md-remediation.yaml
246248
$(KUSTOMIZE) build $(OCI_TEMPLATES)/v1beta1/cluster-template-kcp-remediation --load_restrictor LoadRestrictionsNone > $(OCI_TEMPLATES)/v1beta1/cluster-template-kcp-remediation.yaml
@@ -256,7 +258,7 @@ generate-e2e-templates: kustomize
256258
.PHONY: test-e2e-run
257259
test-e2e-run: generate-e2e-templates ginkgo $(ENVSUBST) ## Run e2e tests
258260
envsubst < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \
259-
$(GINKGO) -v -trace -tags=e2e -focus="$(GINKGO_FOCUS)" -skip="$(GINKGO_SKIP)" -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) $(GINKGO_ARGS) ./test/e2e -- \
261+
$(GINKGO) -v -trace -tags=e2e -focus="$(GINKGO_FOCUS)" -skip=$(GINKGO_SKIP) -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR) $(GINKGO_ARGS) ./test/e2e -- \
260262
-e2e.artifacts-folder="$(ARTIFACTS)" \
261263
-e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \
262264
-e2e.skip-resource-cleanup=$(SKIP_CLEANUP) -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER) $(E2E_ARGS)

api/v1beta1/ocicluster_types.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,10 @@ type OCIClusterSpec struct {
4848
// Compartment to create the cluster network.
4949
CompartmentId string `mandatory:"true" json:"compartmentId"`
5050

51+
// Region the cluster operates in. It must be one of available regions in Region Identifier format.
52+
// See https://docs.oracle.com/en-us/iaas/Content/General/Concepts/regions.htm
53+
Region string `json:"region,omitempty"`
54+
5155
// ControlPlaneEndpoint represents the endpoint used to communicate with the control plane.
5256
// +optional
5357
ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"`

cloud/scope/clients.go

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
/*
2+
Copyright (c) 2022, Oracle and/or its affiliates.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package scope
18+
19+
import (
20+
"sync"
21+
22+
"github.com/go-logr/logr"
23+
"github.com/oracle/cluster-api-provider-oci/cloud/services/compute"
24+
identityClient "github.com/oracle/cluster-api-provider-oci/cloud/services/identity"
25+
nlb "github.com/oracle/cluster-api-provider-oci/cloud/services/networkloadbalancer"
26+
"github.com/oracle/cluster-api-provider-oci/cloud/services/vcn"
27+
"github.com/oracle/oci-go-sdk/v63/common"
28+
"github.com/oracle/oci-go-sdk/v63/core"
29+
"github.com/oracle/oci-go-sdk/v63/identity"
30+
"github.com/oracle/oci-go-sdk/v63/networkloadbalancer"
31+
"github.com/pkg/errors"
32+
"k8s.io/klog/v2/klogr"
33+
)
34+
35+
// OCIClients is the struct of all the needed OCI clients
36+
type OCIClients struct {
37+
ComputeClient compute.ComputeClient
38+
VCNClient vcn.Client
39+
LoadBalancerClient nlb.NetworkLoadBalancerClient
40+
IdentityClient identityClient.Client
41+
}
42+
43+
// ClientProvider defines the regional clients
44+
type ClientProvider struct {
45+
*logr.Logger
46+
ociClients map[string]OCIClients
47+
ociClientsLock *sync.RWMutex
48+
ociAuthConfigProvider common.ConfigurationProvider
49+
}
50+
51+
// NewClientProvider builds the ClientProvider with a client for the given region
52+
func NewClientProvider(ociAuthConfigProvider common.ConfigurationProvider) (*ClientProvider, error) {
53+
log := klogr.New()
54+
55+
if ociAuthConfigProvider == nil {
56+
return nil, errors.New("ConfigurationProvider can not be nil")
57+
}
58+
59+
provider := ClientProvider{
60+
Logger: &log,
61+
ociAuthConfigProvider: ociAuthConfigProvider,
62+
ociClients: map[string]OCIClients{},
63+
ociClientsLock: new(sync.RWMutex),
64+
}
65+
66+
return &provider, nil
67+
}
68+
69+
// GetOrBuildClient if the OCIClients exist for the region they are returned, if not clients will build them
70+
func (c *ClientProvider) GetOrBuildClient(region string) (OCIClients, error) {
71+
if len(region) <= 0 {
72+
return OCIClients{}, errors.New("ClientProvider.GetOrBuildClient region can not be empty")
73+
}
74+
75+
c.ociClientsLock.RLock()
76+
clients, regionalClientsExists := c.ociClients[region]
77+
c.ociClientsLock.RUnlock()
78+
79+
if regionalClientsExists {
80+
return clients, nil
81+
}
82+
83+
c.ociClientsLock.Lock()
84+
defer c.ociClientsLock.Unlock()
85+
regionalClient, err := createClients(region, c.ociAuthConfigProvider, c.Logger)
86+
if err != nil {
87+
return regionalClient, err
88+
}
89+
c.ociClients[region] = regionalClient
90+
91+
return regionalClient, nil
92+
}
93+
94+
func createClients(region string, oCIAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (OCIClients, error) {
95+
vcnClient, err := createVncClient(region, oCIAuthConfigProvider, logger)
96+
lbClient, err := createLbClient(region, oCIAuthConfigProvider, logger)
97+
identityClient, err := createIdentityClient(region, oCIAuthConfigProvider, logger)
98+
computeClient, err := createComputeClient(region, oCIAuthConfigProvider, logger)
99+
100+
if err != nil {
101+
return OCIClients{}, err
102+
}
103+
104+
return OCIClients{
105+
VCNClient: vcnClient,
106+
LoadBalancerClient: lbClient,
107+
IdentityClient: identityClient,
108+
ComputeClient: computeClient,
109+
}, err
110+
}
111+
112+
func createVncClient(region string, ociAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (*core.VirtualNetworkClient, error) {
113+
vcnClient, err := core.NewVirtualNetworkClientWithConfigurationProvider(ociAuthConfigProvider)
114+
if err != nil {
115+
logger.Error(err, "unable to create OCI VCN Client")
116+
return nil, err
117+
}
118+
vcnClient.SetRegion(region)
119+
120+
return &vcnClient, nil
121+
}
122+
123+
func createLbClient(region string, ociAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (*networkloadbalancer.NetworkLoadBalancerClient, error) {
124+
lbClient, err := networkloadbalancer.NewNetworkLoadBalancerClientWithConfigurationProvider(ociAuthConfigProvider)
125+
if err != nil {
126+
logger.Error(err, "unable to create OCI LB Client")
127+
return nil, err
128+
}
129+
lbClient.SetRegion(region)
130+
131+
return &lbClient, nil
132+
}
133+
134+
func createIdentityClient(region string, ociAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (*identity.IdentityClient, error) {
135+
identityClient, err := identity.NewIdentityClientWithConfigurationProvider(ociAuthConfigProvider)
136+
if err != nil {
137+
logger.Error(err, "unable to create OCI Identity Client")
138+
return nil, err
139+
}
140+
identityClient.SetRegion(region)
141+
142+
return &identityClient, nil
143+
}
144+
145+
func createComputeClient(region string, ociAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (*core.ComputeClient, error) {
146+
computeClient, err := core.NewComputeClientWithConfigurationProvider(ociAuthConfigProvider)
147+
if err != nil {
148+
logger.Error(err, "unable to create OCI Compute Client")
149+
return nil, err
150+
}
151+
computeClient.SetRegion(region)
152+
153+
return &computeClient, nil
154+
}

cloud/scope/clients_mock.go

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
/*
2+
Copyright (c) 2022, Oracle and/or its affiliates.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package scope
18+
19+
import (
20+
"bytes"
21+
"crypto/rand"
22+
"crypto/rsa"
23+
"crypto/x509"
24+
"encoding/pem"
25+
"fmt"
26+
"sync"
27+
28+
"github.com/oracle/cluster-api-provider-oci/cloud/config"
29+
"github.com/oracle/cluster-api-provider-oci/cloud/services/compute"
30+
"github.com/oracle/cluster-api-provider-oci/cloud/services/vcn"
31+
"github.com/oracle/oci-go-sdk/v63/identity"
32+
"github.com/oracle/oci-go-sdk/v63/networkloadbalancer"
33+
"k8s.io/klog/v2/klogr"
34+
)
35+
36+
type MockOCIClients struct {
37+
VCNClient vcn.Client
38+
ComputeClient compute.ComputeClient
39+
LoadBalancerClient *networkloadbalancer.NetworkLoadBalancerClient
40+
IdentityClient *identity.IdentityClient
41+
}
42+
43+
var (
44+
MockTestRegion = "us-lexington-1"
45+
)
46+
47+
func MockNewClientProvider(mockClients MockOCIClients) (*ClientProvider, error) {
48+
49+
clientsInject := map[string]OCIClients{MockTestRegion: {
50+
VCNClient: mockClients.VCNClient,
51+
LoadBalancerClient: mockClients.LoadBalancerClient,
52+
IdentityClient: mockClients.IdentityClient,
53+
ComputeClient: mockClients.ComputeClient,
54+
}}
55+
56+
authConfig, err := MockAuthConfig()
57+
if err != nil {
58+
return nil, err
59+
}
60+
61+
ociAuthConfigProvider, err := config.NewConfigurationProvider(&authConfig)
62+
if err != nil {
63+
fmt.Printf("expected ociAuthConfigProvider to be created %s \n", err)
64+
return nil, err
65+
}
66+
log := klogr.New()
67+
clientProvider := ClientProvider{
68+
Logger: &log,
69+
ociClients: clientsInject,
70+
ociClientsLock: new(sync.RWMutex),
71+
ociAuthConfigProvider: ociAuthConfigProvider,
72+
}
73+
74+
return &clientProvider, nil
75+
}
76+
77+
func MockAuthConfig() (config.AuthConfig, error) {
78+
privateKey, err := generatePrivateKeyPEM()
79+
if err != nil {
80+
fmt.Println("error generating a private key")
81+
return config.AuthConfig{}, err
82+
}
83+
84+
authConfig := config.AuthConfig{
85+
UseInstancePrincipals: false,
86+
Region: MockTestRegion,
87+
Fingerprint: "mock-finger-print",
88+
PrivateKey: privateKey,
89+
UserID: "ocid1.tenancy.oc1..<unique_ID>",
90+
TenancyID: "ocid1.tenancy.oc1..<unique_ID>",
91+
}
92+
93+
return authConfig, nil
94+
}
95+
96+
func generatePrivateKeyPEM() (string, error) {
97+
privateKey, err := rsa.GenerateKey(rand.Reader, 1024)
98+
if err != nil {
99+
return "", err
100+
}
101+
102+
privateKeyBytes := x509.MarshalPKCS1PrivateKey(privateKey)
103+
privateKeyBlock := &pem.Block{
104+
Type: "RSA PRIVATE KEY",
105+
Bytes: privateKeyBytes,
106+
}
107+
108+
var privateKeyBuf bytes.Buffer
109+
err = pem.Encode(&privateKeyBuf, privateKeyBlock)
110+
if err != nil {
111+
fmt.Printf("error when encode private pem: %s \n", err)
112+
return "", err
113+
}
114+
115+
return privateKeyBuf.String(), err
116+
}

0 commit comments

Comments
 (0)