Skip to content

Commit 8f53e7e

Browse files
mrogmrog
committed
Merge branch 'feature/add-e2e-tests' into e2e_2_clusters
2 parents e25c767 + 63bab26 commit 8f53e7e

File tree

9 files changed

+444
-473
lines changed

9 files changed

+444
-473
lines changed

‎ATTRIBUTION.txt

Lines changed: 174 additions & 467 deletions
Large diffs are not rendered by default.

‎pkg/cloud/network.go

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ type NetworkIface interface {
3030
ResolveNetworkStatuses(*capcv1.CloudStackCluster) error
3131
ResolveNetwork(*capcv1.CloudStackCluster, *capcv1.Network) error
3232
CreateIsolatedNetwork(*capcv1.CloudStackCluster) error
33-
OpenFirewallRules(*capcv1.CloudStackCluster) error
33+
OpenFirewallRules(networkID string) error
3434
FetchPublicIP(*capcv1.CloudStackCluster) (*cloudstack.PublicIpAddress, error)
3535
ResolveLoadBalancerRuleDetails(*capcv1.CloudStackCluster) error
3636
GetOrCreateLoadBalancerRule(*capcv1.CloudStackCluster) error
@@ -146,6 +146,10 @@ func (c *client) CreateIsolatedNetwork(csCluster *capcv1.CloudStackCluster) (ret
146146
return err
147147
}
148148

149+
if err := c.OpenFirewallRules(zoneStatus.Network.ID); err != nil {
150+
return err
151+
}
152+
149153
return nil
150154
}
151155

@@ -268,8 +272,8 @@ func (c *client) AssociatePublicIPAddress(csCluster *capcv1.CloudStackCluster) (
268272
return nil
269273
}
270274

271-
func (c *client) OpenFirewallRules(csCluster *capcv1.CloudStackCluster) (retErr error) {
272-
p := c.cs.Firewall.NewCreateEgressFirewallRuleParams(csCluster.Status.PublicIPNetworkID, NetworkProtocolTCP)
275+
func (c *client) OpenFirewallRules(networkID string) (retErr error) {
276+
p := c.cs.Firewall.NewCreateEgressFirewallRuleParams(networkID, NetworkProtocolTCP)
273277
_, retErr = c.cs.Firewall.CreateEgressFirewallRule(p)
274278
if retErr != nil && strings.Contains(strings.ToLower(retErr.Error()), "there is already") { // Already a firewall rule here.
275279
retErr = nil

‎pkg/cloud/network_test.go

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,10 @@ var _ = Describe("Network", func() {
127127
PublicIpAddresses: []*csapi.PublicIpAddress{{Id: dummies.PublicIPID, Ipaddress: "fakeIP"}}}, nil)
128128
as.EXPECT().NewAssociateIpAddressParams().Return(&csapi.AssociateIpAddressParams{})
129129
as.EXPECT().AssociateIpAddress(gomock.Any())
130+
fs.EXPECT().NewCreateEgressFirewallRuleParams(dummies.ISONet1.ID, cloud.NetworkProtocolTCP).
131+
Return(&csapi.CreateEgressFirewallRuleParams{})
132+
fs.EXPECT().CreateEgressFirewallRule(&csapi.CreateEgressFirewallRuleParams{}).
133+
Return(&csapi.CreateEgressFirewallRuleResponse{}, nil)
130134

131135
// Will add cluster tag once to Network and once to PublicIP.
132136
createdByResponse := &csapi.ListTagsResponse{Tags: []*csapi.Tag{{Key: cloud.CreatedByCAPCTagName, Value: "1"}}}
@@ -159,7 +163,7 @@ var _ = Describe("Network", func() {
159163
fs.EXPECT().CreateEgressFirewallRule(&csapi.CreateEgressFirewallRuleParams{}).
160164
Return(&csapi.CreateEgressFirewallRuleResponse{}, nil)
161165

162-
Ω(client.OpenFirewallRules(dummies.CSCluster)).Should(Succeed())
166+
Ω(client.OpenFirewallRules(dummies.ISONet1.ID)).Should(Succeed())
163167
})
164168
})
165169

@@ -174,7 +178,7 @@ var _ = Describe("Network", func() {
174178
fs.EXPECT().CreateEgressFirewallRule(&csapi.CreateEgressFirewallRuleParams{}).
175179
Return(&csapi.CreateEgressFirewallRuleResponse{}, errors.New("there is already a rule like this"))
176180

177-
Ω(client.OpenFirewallRules(dummies.CSCluster)).Should(Succeed())
181+
Ω(client.OpenFirewallRules(dummies.ISONet1.ID)).Should(Succeed())
178182
})
179183
})
180184

‎releases/v0.4.0/EVALUATION_DEPLOYMENT.md

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
# CloudStack Cluster API Provider (CAPC) Release v.0.4.0 Evaluation Deployment Guide
2+
3+
This document defines a manual deployment process suitable for evaluating this CAPC release.
4+
5+
## Evaluation Environment Pre-Requisites:
6+
7+
### - A running Kubernetes cluster for hosting CAPC
8+
9+
This should be an easily disposable/re-creatable cluster, such as a locally-running kind (Kuberetes in Docker) cluster.
10+
11+
Your KUBECONFIG file's *current-context* must be set to the cluster you want to use.
12+
13+
### - CAPI clusterctl v1.0.1 (https://github.com/kubernetes-sigs/cluster-api/releases/tag/v1.0.1)
14+
15+
This process has been tested with this version of clusterctl. Subsequent 1.0.x versions should work as well.
16+
17+
### - A CloudStack Environment with the following resources defined
18+
- Zone
19+
- Network
20+
- CAPI-compatible QEMU template (i.e., created with https://github.com/kubernetes-sigs/image-builder)
21+
- Machine Offerings (suitable for running Kubernetes nodes)
22+
- apikey and secretkey for a CloudStack user having domain administrative privileges
23+
- Available ACS IP Address for the k8s Control Plane endpoint (Shared network: available IP address in the network range; isolated network: public IP address)
24+
25+
## Deployment Steps
26+
### Define Identity Environment Variable
27+
28+
An environment variable named CLOUDSTACK_B64ENCODED_SECRET must be defined, containing the base64 encoding of a
29+
cloud-config properties file. This file is of the form:
30+
31+
```
32+
[Global]
33+
api-url = <urlOfCloudStackAPI>
34+
api-key = <cloudstackUserApiKey>
35+
secret-key = <cloudstackUserSecretKey>
36+
```
37+
After defining this in a file named cloud-config, create the environment variable with:
38+
39+
```
40+
export CLOUDSTACK_B64ENCODED_SECRET=$(base64 -w0 -i cloud-config 2>/dev/null || base64 -b 0 -i cloud-config)
41+
```
42+
43+
For security, delete this cloud-config file after creating this environment variable.
44+
45+
### Deploy the supplied container image archive (.tar.gz) to a suitable image registry.
46+
47+
*We use https://github.com/kubernetes-sigs/cluster-api/blob/main/hack/kind-install-for-capd.sh to launch a local
48+
docker registry integrated into a kind cluster for lightweight development and testing.*
49+
50+
- On a computer with docker, load the provided cluster-api-provider-capc.tar.gz to docker:
51+
```
52+
docker load --input cluster-api-provider-capc_v0.4.0.tar.gz
53+
```
54+
55+
This will create image *localhost:5000/cluster-api-provider-cloudstack:v0.4.0* in your local docker. This is suitable
56+
for pushing to a local registry.
57+
58+
- (Optional) Tag this image for your registry.
59+
```
60+
docker tag localhost:5000/cluster-api-provider-cloudstack:v0.4.0 <yourRepoFqdn>/cluster-api-provider-cloudstack:v0.4.0
61+
```
62+
63+
Push it to your registry (localhost:5000 if using local registry)
64+
```
65+
docker push <yourRepoFqdn>/cluster-api-provider-cloudstack:v0.4.0
66+
```
67+
68+
### Create clusterctl configuration files
69+
A cluster-api.zip file has been provided, containing the files and directory structure suitable for configuring
70+
clusterctl to work with this interim release of CAPC. It should be restored under $HOME/.cluster-api. It contains:
71+
72+
```
73+
Archive: /Users/jweite/Dev/cluster-api-cloudstack-v0.4.0-assets/cluster-api.zip
74+
* clusterctl.yaml
75+
* dev-repository/
76+
* dev-repository/infrastructure-cloudstack/
77+
* dev-repository/infrastructure-cloudstack/v0.4.0/
78+
* dev-repository/infrastructure-cloudstack/v0.4.0/cluster-template.yaml
79+
* dev-repository/infrastructure-cloudstack/v0.4.0/cluster-template-managed-ssh.yaml
80+
* dev-repository/infrastructure-cloudstack/v0.4.0/cluster-template-ssh-material.yaml
81+
* dev-repository/infrastructure-cloudstack/v0.4.0/infrastructure-components.yaml
82+
* dev-repository/infrastructure-cloudstack/v0.4.0/metadata.yaml
83+
```
84+
85+
*Note: If you already have a $HOME/.cluster-api we strongly suggest you delete or stash it.*
86+
87+
```
88+
cd ~
89+
mkdir .cluster-api
90+
cd .cluster-api
91+
unzip cluster-api.zip
92+
```
93+
94+
### Edit the clusterctl configuration files
95+
- **clusterctl.yaml:** in the *url* attribute replace \<USERID\> with your OS user id to form a valid absolute path to infrastructure-components.yaml.
96+
97+
- **dev-repository/infrastructure-cloudstack/v0.4.0/infrastructure-components.yaml:** if you're not using a local registry modify the capc-controller-manager deployment, changing the spec.template.spec.containers[0].image (line 617) to correctly reflect your container registry.
98+
99+
### Deploy CAPI and CAPC to your bootstrap Kubernetes cluster
100+
```
101+
clusterctl init --infrastructure cloudstack
102+
```
103+
104+
### Generate a manifest for the CAPI custom resources needed to allocate a workload cluster.
105+
106+
*Set the below environment variables as appropriate for your CloudStack environment.*
107+
108+
```
109+
CLOUDSTACK_ZONE_NAME=<MyZoneName> \
110+
CLOUDSTACK_NETWORK_NAME=<MyNetworkName> \
111+
CLOUDSTACK_TEMPLATE_NAME=<MyTemplateName> \
112+
CLOUDSTACK_CONTROL_PLANE_MACHINE_OFFERING=<MyServiceOfferingName> \
113+
CONTROL_PLANE_MACHINE_COUNT=1 \
114+
CLOUDSTACK_WORKER_MACHINE_OFFERING=<MyServiceOfferingName> \
115+
WORKER_MACHINE_COUNT=1 \
116+
CLUSTER_ENDPOINT_IP=<AvailableSharedOrPublicIP> \
117+
CLUSTER_ENDPOINT_PORT=6443 \
118+
KUBERNETES_VERSION=<KubernetesVersionOnTheImage> \
119+
CLUSTER_NAME=<MyClusterName> \
120+
clusterctl generate yaml --from ~/.cluster-api/dev-repository/infrastructure-cloudstack/v0.4.0/cluster-template.yaml > clusterTemplate.yaml
121+
```
122+
123+
### Review the generated clusterTemplate.yaml and adjust as necessary
124+
125+
126+
### Provision your workload cluster
127+
128+
```
129+
kubectl apply -f clusterTemplate.yaml
130+
```
131+
132+
Provisioning can take several minutes to complete. You will see a control plane VM created in CloudStack pretty quickly,
133+
but it takes a while for it to complete its cloud-init to install Kubernetes and become a functioning control plane.
134+
Allocation of the worker node(s) (with *md* in their VM names) won't occur until the control plane is operational.
135+
136+
You can monitor the CAPC controller as it conducts the provisioning process with:
137+
```
138+
# Get the full name of the CAPC controller pod
139+
kubectl -n capc-system get pods
140+
141+
# Tail its logs
142+
kubectl -n capc-system log -f <CAPCcontrollerPodFullName>
143+
```
144+
145+
### Fetch a kubeconfig to access your cluster
146+
```
147+
clusterctl get kubeconfig <clusterName> > <clusterName>_kubeconfig
148+
```
149+
150+
You can then either export a KUBECONFIG environment variable pointing to this file, or use kubectl's --kubeconfig=<filePath>
151+
flag.
152+
```
153+
export KUBECONFIG=<clusterName>_kubeconfig
154+
```
155+
156+
### Examine the provisioned Kubernetes Cluster's nodes
157+
```
158+
kubectl get nodes
159+
```
160+
Expect to see a control plane and a worker node reported by Kubernetes. Neither will report that they are ready
161+
because no CNI is installed yet.
162+
163+
### Install Cilium CNI
164+
```
165+
cilium install
166+
```
167+
The above command presumes that the cilium installer is present on the local workstation.
168+
169+
It will take a minute while it waits for cilium to become active.
170+
171+
### Confirm that Cluster is Ready for Work
172+
```
173+
kubectl get nodes
174+
```
175+
Expect now to see both nodes list as ready.
176+
177+
### Conclusion
178+
At this point the workload cluster is ready to accept workloads. Use it in the usual way via the kubeconfig generated
179+
earlier
180+
181+
### Cluster Deletion
182+
As mentioned in the preface, CAPC is not yet able to delete workload cluster. To do so manually we recommend
183+
simply tearing-down the kind bootstrap cluster, and then manually deleting the CloudStack VMs created for it
184+
using the CloudStack UI, API or similar facilities.

‎releases/v0.4.0/RELEASE_NOTES.md

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
# Cluster API Provider for Cloudstack (CAPC) Release Notes
2+
3+
## Version 0.4.0
4+
5+
These Release Notes are for the customer downloading and deploying CAPC private Version 0.4.0 released on 03/11/2022.
6+
7+
### This release extends the v0.3.0 release of CAPC with:
8+
9+
* v1beta1 API declared
10+
* Support for distributing cluster virtual machines across multiple CloudStack Zones/Networks
11+
* Enablement of the CAPC controller metrics port
12+
* Improved cleanup of CloudStack network components allocated as part of a cluster upon cluster deletion.
13+
* Accelerated cluster deletion through concurrent VM deletion
14+
* Node names match machine names
15+
* Support for disabling TLS certificate validation for CloudStack connections (cloud_config verify-ssl boolean parameter, true by default).
16+
* Example templates for implementing CAPI Machine Health Checking / Remediation of CAPC clusters
17+
18+
### TLS Certificates
19+
The default mode of operation for the deployed Kubernetes cluster components is to use self-signed certificates. Options exist for use of an enterprise certificate authority via cert-manager (https://cert-manager.io/docs/configuration/). Detailed configuration of this component is outside the scope of this release.
20+
21+
### Pre-conditions
22+
23+
* The following pre-conditions must be met for CAPC to operate as designed.
24+
* A functional CloudStack 4.14 or 4.16 deployment
25+
* The CloudStack account used by CAPC must have domain administrator privileges or be otherwise appropriately privileged to execute the API calls specified in the below CAPC CloudStack API Calls document link.
26+
* Zone(s) and Network(s) must be pre-created and available to CAPC prior to CreateCluster API call.
27+
* A VM template suitable for implementing a Kubernetes node with kubeadm must be available in CloudStack.
28+
* The software has been tested with RHEL-8 images created with CAPI Image-builder.
29+
* Machine offerings suitable for running Kubernetes nodes must be available in CloudStack
30+
* When using CloudStack Shared Networks, an unused IP address in the shared network’s address range must be available for the Kubernetes Control Plane for each cluster, upon which it will be exposed.
31+
32+
### Release Assets :
33+
34+
* cluster-api-provider-cloudstack-v0.4.0.tar.gz: container image of the CAPC controller
35+
* shasum.txt containing checksum for the released cluster-api-provider-cloudstack-v0.4.0.tar.gz
36+
* cluster-api.zip: configuration files for clusterctl
37+
* infrastructure-components.yaml
38+
* metadata.yaml
39+
* cluster-template.yaml
40+
* cluster-template-ssh.yaml
41+
* EVALUATION_DEPLOYMENT.md: instructions for manual deployment of this interim release for evaluation via clusterctl.
42+
* security_findings.csv: results of package security scan
43+
44+
45+
### Known Issues :
46+
47+
* Cluster upgrade is not supported when the controlPlaneEndpoint is defined to be an IP address in a shared network.
48+
49+
### Future Scope/Features
50+
51+
* Accelerated remediation of VM state drift

‎releases/v0.4.0/clusterApi.zip

10.7 KB
Binary file not shown.

‎releases/v0.4.0/security_findings.csv

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
Component,Vulnerability,AWS Assessment
2+
cloud.google.com/go/storage:1.10.0,https://nvd.nist.gov/vuln/search/results?form_type=Advanced&results_type=overview&search_type=all&cpe_vendor=cpe%3A%2F%3Astorage_project&cpe_product=cpe%3A%2F%3Astorage_project%3Astorage&cpe_version=cpe%3A%2F%3Astorage_project%3Astorage%3A1.10.0,No exploitable issue. This finding only affects applications unpacking container Image manifests.
3+
github.com/coreos/etcd:3.3.13+incompatible,https://nvd.nist.gov/vuln/search/results?form_type=Advanced&results_type=overview&search_type=all&cpe_vendor=cpe%3A%2F%3Aetcd&cpe_product=cpe%3A%2F%3Aetcd%3Aetcd&cpe_version=cpe%3A%2F%3Aetcd%3Aetcd%3A3.3.13,"No exploitable issue. etcd is unused in Kubernetes CAPI controllers, only the Kubernetes API server interacts with an etcd database."
4+
github.com/docker/distribution:2.7.1+incompatible,cpe:2.3:a:docker:docker:2.7.1,No exploitable issue. The Docker API and client are unused in a Kubernetes CAPI controller.
5+
github.com/grpc-ecosystem/go-grpc-middleware:1.3.0,https://nvd.nist.gov/vuln/search/results?form_type=Advanced&results_type=overview&search_type=all&cpe_vendor=cpe%3A%2F%3Agrpc&cpe_product=cpe%3A%2F%3Agrpc%3Agrpc&cpe_version=cpe%3A%2F%3Agrpc%3Agrpc%3A1.3.0,No exploitable issue. Kubernetes controllers do not make or issue gRPC calls.
6+
github.com/grpc-ecosystem/go-grpc-prometheus:1.2.0,https://nvd.nist.gov/vuln/search/results?form_type=Advanced&results_type=overview&search_type=all&cpe_vendor=cpe%3A%2F%3Agrpc&cpe_product=cpe%3A%2F%3Agrpc%3Agrpc&cpe_version=cpe%3A%2F%3Agrpc%3Agrpc%3A1.2.0,No exploitable issue. Kubernetes controllers do not make or issue gRPC calls.
7+
github.com/grpc-ecosystem/grpc-gateway:1.16.0,https://nvd.nist.gov/vuln/search/results?form_type=Advanced&results_type=overview&search_type=all&cpe_vendor=cpe%3A%2F%3Agrpc&cpe_product=cpe%3A%2F%3Agrpc%3Agrpc&cpe_version=cpe%3A%2F%3Agrpc%3Agrpc%3A1.16.0,No exploitable issue. Kubernetes controllers do not make or issue gRPC calls.
8+
github.com/hashicorp/consul/api:1.1.0,cpe:2.3:a:hashicorp:consul:1.1.0,No exploitable issue. Consul is unused by a Kubernetes CAPI controller.
9+
github.com/hashicorp/consul/sdk:0.1.1,cpe:2.3:a:hashicorp:consul:0.1.1,No exploitable issue. Consul is unused by a Kubernetes CAPI controller.
10+
github.com/matttproud/golang_protobuf_extensions:1.0.2-0.20181231171920-c182affec369,https://nvd.nist.gov/vuln/search/results?form_type=Advanced&results_type=overview&search_type=all&cpe_vendor=cpe%3A%2F%3Agolang&cpe_product=cpe%3A%2F%3Agolang%3Aprotobuf&cpe_version=cpe%3A%2F%3Agolang%3Aprotobuf%3A1.0.2.0.20181231171920.c182.fec369,No exploitable issue. Kubernetes controllers do not make or issue gRPC calls.
11+
github.com/prometheus/client_golang:1.11.0,cpe:2.3:a:prometheus:prometheus:1.11.0,No exploitable issue. The mentioned vulnerability is related to the Prometheus UI.
12+
github.com/prometheus/client_model:0.2.0,cpe:2.3:a:prometheus:prometheus:0.2.0,No exploitable issue. The mentioned vulnerability is related to the Prometheus UI.
13+
github.com/prometheus/common:0.32.1,cpe:2.3:a:prometheus:prometheus:0.32.1,No exploitable issue. The mentioned vulnerability is related to the Prometheus UI.
14+
github.com/prometheus/procfs:0.7.3,cpe:2.3:a:prometheus:prometheus:0.7.3,No exploitable issue. The mentioned vulnerability is related to the Prometheus UI.
15+
github.com/prometheus/tsdb:0.7.1,cpe:2.3:a:prometheus:prometheus:0.7.1,No exploitable issue. The mentioned vulnerability is related to the Prometheus UI.
16+
github.com/tmc/grpc-websocket-proxy:0.0.0-20201229170055-e5319fda7802 ,https://nvd.nist.gov/vuln/search/results?form_type=Advanced&results_type=overview&search_type=all&cpe_vendor=cpe%3A%2F%3Agrpc&cpe_product=cpe%3A%2F%3Agrpc%3Agrpc&cpe_version=cpe%3A%2F%3Agrpc%3Agrpc%3A0.0.0.20201229170055.e5319.fda7802,No exploitable issue. Kubernetes controllers do not make or issue gRPC calls.
17+
go.etcd.io/etcd/client/v2:2.305.0,cpe:2.3:a:etcd:etcd:2.305.0,"No exploitable issue. etcd is unused in Kubernetes CAPI controllers, only the Kubernetes API server interacts with an etcd database."
18+
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc:0.20.0 ,https://nvd.nist.gov/vuln/search/results?form_type=Advanced&results_type=overview&search_type=all&cpe_vendor=cpe%3A%2F%3Agrpc&cpe_product=cpe%3A%2F%3Agrpc%3Agrpc&cpe_version=cpe%3A%2F%3Agrpc%3Agrpc%3A0.20.0,No exploitable issue. Kubernetes controllers do not make or issue gRPC calls.
19+
google.golang.org/grpc/cmd/protoc-gen-go-grpc:1.1.0,https://nvd.nist.gov/vuln/search/results?form_type=Advanced&results_type=overview&search_type=all&cpe_vendor=cpe%3A%2F%3Agrpc&cpe_product=cpe%3A%2F%3Agrpc%3Agrpc&cpe_version=cpe%3A%2F%3Agrpc%3Agrpc%3A1.1.0,No exploitable issue. Kubernetes controllers do not make or issue gRPC calls.
20+
github.com/docker/distribution,https://github.com/advisories/GHSA-qq97-vm5h-rrhg,No exploitable issue. Vulnerable code is not actually used.

‎releases/v0.4.0/sha1sum.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
3ec165162ccd259ffcc6ff83bceeb64cc047a173 cluster-api-provider-cloudstack-v0.4.0.tar.gz

‎test/e2e/invalid_resource.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ func InvalidResourceSpec(ctx context.Context, inputGetter func() CommonSpecInput
8686
})
8787

8888
It("Should fail due to the compute resources are not sufficient for the specified offering [TC8]", func() {
89-
testInvalidResource(ctx, input, "insufficient-compute-resources", "CloudStackMachine VM in error state. Deleting associated Machine.")
89+
testInvalidResource(ctx, input, "insufficient-compute-resources", "Unable to create a deployment for VM")
9090
})
9191

9292
AfterEach(func() {

0 commit comments

Comments
 (0)