Skip to content

Commit 25e4ee7

Browse files
hamzyhamzy
committed
PowerVS: Add destroy Power Server subnets
When we cleanup the Power Server DHCP server, it sometimes leaves behind its backing subnet. If this is left around it will cause network problems for the next cluster. Therefore, search and clean them up!
1 parent 19d5337 commit 25e4ee7

File tree

4 files changed

+182
-27
lines changed

4 files changed

+182
-27
lines changed

pkg/destroy/powervs/subnet.go renamed to pkg/destroy/powervs/cloud-subnet.go

Lines changed: 25 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -13,18 +13,18 @@ import (
1313
"k8s.io/apimachinery/pkg/util/wait"
1414
)
1515

16-
const subnetTypeName = "subnet"
16+
const cloudSubnetTypeName = "cloudSubnet"
1717

18-
// listSubnets lists subnets in the cloud.
19-
func (o *ClusterUninstaller) listSubnets() (cloudResources, error) {
20-
o.Logger.Debugf("Listing Subnets")
18+
// listCloudSubnets lists subnets in the VPC cloud.
19+
func (o *ClusterUninstaller) listCloudSubnets() (cloudResources, error) {
20+
o.Logger.Debugf("Listing virtual Cloud Subnets")
2121

2222
ctx, cancel := o.contextWithTimeout()
2323
defer cancel()
2424

2525
select {
2626
case <-ctx.Done():
27-
o.Logger.Debugf("listSubnets: case <-ctx.Done()")
27+
o.Logger.Debugf("listCloudSubnets: case <-ctx.Done()")
2828
return nil, o.Context.Err() // we're cancelled, abort
2929
default:
3030
}
@@ -42,27 +42,27 @@ func (o *ClusterUninstaller) listSubnets() (cloudResources, error) {
4242
for _, subnet := range subnets.Subnets {
4343
if strings.Contains(*subnet.Name, o.InfraID) {
4444
foundOne = true
45-
o.Logger.Debugf("listSubnets: FOUND: %s, %s", *subnet.ID, *subnet.Name)
45+
o.Logger.Debugf("listCloudSubnets: FOUND: %s, %s", *subnet.ID, *subnet.Name)
4646
result = append(result, cloudResource{
4747
key: *subnet.ID,
4848
name: *subnet.Name,
4949
status: "",
50-
typeName: subnetTypeName,
50+
typeName: cloudSubnetTypeName,
5151
id: *subnet.ID,
5252
})
5353
}
5454
}
5555
if !foundOne {
56-
o.Logger.Debugf("listSubnets: NO matching subnet against: %s", o.InfraID)
56+
o.Logger.Debugf("listCloudSubnets: NO matching subnet against: %s", o.InfraID)
5757
for _, subnet := range subnets.Subnets {
58-
o.Logger.Debugf("listSubnets: subnet: %s", *subnet.Name)
58+
o.Logger.Debugf("listCloudSubnets: subnet: %s", *subnet.Name)
5959
}
6060
}
6161

6262
return cloudResources{}.insert(result...), nil
6363
}
6464

65-
func (o *ClusterUninstaller) deleteSubnet(item cloudResource) error {
65+
func (o *ClusterUninstaller) deleteCloudSubnet(item cloudResource) error {
6666
var getOptions *vpcv1.GetSubnetOptions
6767
var response *core.DetailedResponse
6868
var err error
@@ -72,7 +72,7 @@ func (o *ClusterUninstaller) deleteSubnet(item cloudResource) error {
7272

7373
select {
7474
case <-ctx.Done():
75-
o.Logger.Debugf("deleteSubnet: case <-ctx.Done()")
75+
o.Logger.Debugf("deleteCloudSubnet: case <-ctx.Done()")
7676
return o.Context.Err() // we're cancelled, abort
7777
default:
7878
}
@@ -87,7 +87,7 @@ func (o *ClusterUninstaller) deleteSubnet(item cloudResource) error {
8787
return nil
8888
}
8989
if err != nil && response != nil && response.StatusCode == gohttp.StatusInternalServerError {
90-
o.Logger.Infof("deleteSubnet: internal server error")
90+
o.Logger.Infof("deleteCloudSubnet: internal server error")
9191
return nil
9292
}
9393

@@ -103,10 +103,10 @@ func (o *ClusterUninstaller) deleteSubnet(item cloudResource) error {
103103
return nil
104104
}
105105

106-
// destroySubnets removes all subnet resources that have a name prefixed
106+
// destroyCloudSubnets removes all subnet resources that have a name prefixed
107107
// with the cluster's infra ID.
108-
func (o *ClusterUninstaller) destroySubnets() error {
109-
firstPassList, err := o.listSubnets()
108+
func (o *ClusterUninstaller) destroyCloudSubnets() error {
109+
firstPassList, err := o.listCloudSubnets()
110110
if err != nil {
111111
return err
112112
}
@@ -115,15 +115,15 @@ func (o *ClusterUninstaller) destroySubnets() error {
115115
return nil
116116
}
117117

118-
items := o.insertPendingItems(subnetTypeName, firstPassList.list())
118+
items := o.insertPendingItems(cloudSubnetTypeName, firstPassList.list())
119119

120120
ctx, cancel := o.contextWithTimeout()
121121
defer cancel()
122122

123123
for _, item := range items {
124124
select {
125125
case <-ctx.Done():
126-
o.Logger.Debugf("destroySubnets: case <-ctx.Done()")
126+
o.Logger.Debugf("destroyCloudSubnets: case <-ctx.Done()")
127127
return o.Context.Err() // we're cancelled, abort
128128
default:
129129
}
@@ -134,23 +134,23 @@ func (o *ClusterUninstaller) destroySubnets() error {
134134
Cap: leftInContext(ctx),
135135
Steps: math.MaxInt32}
136136
err = wait.ExponentialBackoffWithContext(ctx, backoff, func(context.Context) (bool, error) {
137-
err2 := o.deleteSubnet(item)
137+
err2 := o.deleteCloudSubnet(item)
138138
if err2 == nil {
139139
return true, err2
140140
}
141141
o.errorTracker.suppressWarning(item.key, err2, o.Logger)
142142
return false, err2
143143
})
144144
if err != nil {
145-
o.Logger.Fatal("destroySubnets: ExponentialBackoffWithContext (destroy) returns ", err)
145+
o.Logger.Fatal("destroyCloudSubnets: ExponentialBackoffWithContext (destroy) returns ", err)
146146
}
147147
}
148148

149-
if items = o.getPendingItems(subnetTypeName); len(items) > 0 {
149+
if items = o.getPendingItems(cloudSubnetTypeName); len(items) > 0 {
150150
for _, item := range items {
151-
o.Logger.Debugf("destroySubnets: found %s in pending items", item.name)
151+
o.Logger.Debugf("destroyCloudSubnets: found %s in pending items", item.name)
152152
}
153-
return fmt.Errorf("destroySubnets: %d undeleted items pending", len(items))
153+
return fmt.Errorf("destroyCloudSubnets: %d undeleted items pending", len(items))
154154
}
155155

156156
backoff := wait.Backoff{
@@ -159,7 +159,7 @@ func (o *ClusterUninstaller) destroySubnets() error {
159159
Cap: leftInContext(ctx),
160160
Steps: math.MaxInt32}
161161
err = wait.ExponentialBackoffWithContext(ctx, backoff, func(context.Context) (bool, error) {
162-
secondPassList, err2 := o.listSubnets()
162+
secondPassList, err2 := o.listCloudSubnets()
163163
if err2 != nil {
164164
return false, err2
165165
}
@@ -168,12 +168,12 @@ func (o *ClusterUninstaller) destroySubnets() error {
168168
return true, nil
169169
}
170170
for _, item := range secondPassList {
171-
o.Logger.Debugf("destroySubnets: found %s in second pass", item.name)
171+
o.Logger.Debugf("destroyCloudSubnets: found %s in second pass", item.name)
172172
}
173173
return false, nil
174174
})
175175
if err != nil {
176-
o.Logger.Fatal("destroySubnets: ExponentialBackoffWithContext (list) returns ", err)
176+
o.Logger.Fatal("destroyCloudSubnets: ExponentialBackoffWithContext (list) returns ", err)
177177
}
178178

179179
return nil

‎pkg/destroy/powervs/power-subnet.go‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
package powervs
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"math"
7+
"strings"
8+
"time"
9+
10+
"k8s.io/apimachinery/pkg/util/wait"
11+
)
12+
13+
const powerSubnetTypeName = "powerSubnet"
14+
15+
// listPowerSubnets lists subnets in the Power Server.
16+
func (o *ClusterUninstaller) listPowerSubnets() (cloudResources, error) {
17+
o.Logger.Debugf("Listing Power Server Subnets")
18+
19+
if o.instanceClient == nil {
20+
o.Logger.Infof("Skipping deleting Power service subnets because no service instance was found")
21+
result := []cloudResource{}
22+
return cloudResources{}.insert(result...), nil
23+
}
24+
25+
networks, err := o.networkClient.GetAll()
26+
if err != nil {
27+
o.Logger.Warnf("Error networkClient.GetAll: %v", err)
28+
return nil, err
29+
}
30+
31+
result := []cloudResource{}
32+
for _, network := range networks.Networks {
33+
if strings.Contains(*network.Name, o.InfraID) {
34+
o.Logger.Debugf("listPowerSubnets: FOUND: %s, %s", *network.NetworkID, *network.Name)
35+
result = append(result, cloudResource{
36+
key: *network.NetworkID,
37+
name: *network.Name,
38+
status: "",
39+
typeName: powerSubnetTypeName,
40+
id: *network.NetworkID,
41+
})
42+
}
43+
}
44+
if len(result) == 0 {
45+
o.Logger.Debugf("listPowerSubnets: NO matching subnet against: %s", o.InfraID)
46+
for _, network := range networks.Networks {
47+
o.Logger.Debugf("listPowerSubnets: network: %s", *network.Name)
48+
}
49+
}
50+
51+
return cloudResources{}.insert(result...), nil
52+
}
53+
54+
func (o *ClusterUninstaller) deletePowerSubnet(item cloudResource) error {
55+
if _, err := o.networkClient.Get(item.id); err != nil {
56+
o.deletePendingItems(item.typeName, []cloudResource{item})
57+
o.Logger.Infof("Deleted Power Network %q", item.name)
58+
return nil
59+
}
60+
61+
o.Logger.Debugf("Deleting Power Network %q", item.name)
62+
63+
if err := o.networkClient.Delete(item.id); err != nil {
64+
o.Logger.Infof("Error: o.networkClient.Delete: %q", err)
65+
return err
66+
}
67+
68+
o.deletePendingItems(item.typeName, []cloudResource{item})
69+
o.Logger.Infof("Deleted Power Network %q", item.name)
70+
71+
return nil
72+
}
73+
74+
// destroyPowerSubnets removes all subnet resources that have a name prefixed
75+
// with the cluster's infra ID.
76+
func (o *ClusterUninstaller) destroyPowerSubnets() error {
77+
firstPassList, err := o.listPowerSubnets()
78+
if err != nil {
79+
return err
80+
}
81+
82+
if len(firstPassList.list()) == 0 {
83+
return nil
84+
}
85+
86+
items := o.insertPendingItems(powerSubnetTypeName, firstPassList.list())
87+
88+
ctx, cancel := o.contextWithTimeout()
89+
defer cancel()
90+
91+
for _, item := range items {
92+
select {
93+
case <-ctx.Done():
94+
o.Logger.Debugf("destroyPowerSubnets: case <-ctx.Done()")
95+
return o.Context.Err() // we're cancelled, abort
96+
default:
97+
}
98+
99+
backoff := wait.Backoff{
100+
Duration: 15 * time.Second,
101+
Factor: 1.1,
102+
Cap: leftInContext(ctx),
103+
Steps: math.MaxInt32}
104+
err = wait.ExponentialBackoffWithContext(ctx, backoff, func(context.Context) (bool, error) {
105+
err2 := o.deletePowerSubnet(item)
106+
if err2 == nil {
107+
return true, err2
108+
}
109+
o.errorTracker.suppressWarning(item.key, err2, o.Logger)
110+
return false, err2
111+
})
112+
if err != nil {
113+
o.Logger.Fatal("destroyPowerSubnets: ExponentialBackoffWithContext (destroy) returns ", err)
114+
}
115+
}
116+
117+
if items = o.getPendingItems(powerSubnetTypeName); len(items) > 0 {
118+
for _, item := range items {
119+
o.Logger.Debugf("destroyPowerSubnets: found %s in pending items", item.name)
120+
}
121+
return fmt.Errorf("destroyPowerSubnets: %d undeleted items pending", len(items))
122+
}
123+
124+
backoff := wait.Backoff{
125+
Duration: 15 * time.Second,
126+
Factor: 1.1,
127+
Cap: leftInContext(ctx),
128+
Steps: math.MaxInt32}
129+
err = wait.ExponentialBackoffWithContext(ctx, backoff, func(context.Context) (bool, error) {
130+
secondPassList, err2 := o.listPowerSubnets()
131+
if err2 != nil {
132+
return false, err2
133+
}
134+
if len(secondPassList) == 0 {
135+
// We finally don't see any remaining instances!
136+
return true, nil
137+
}
138+
for _, item := range secondPassList {
139+
o.Logger.Debugf("destroyPowerSubnets: found %s in second pass", item.name)
140+
}
141+
return false, nil
142+
})
143+
if err != nil {
144+
o.Logger.Fatal("destroyPowerSubnets: ExponentialBackoffWithContext (list) returns ", err)
145+
}
146+
147+
return nil
148+
}

‎pkg/destroy/powervs/powervs.go‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ type ClusterUninstaller struct {
9696
jobClient *instance.IBMPIJobClient
9797
keyClient *instance.IBMPIKeyClient
9898
dhcpClient *instance.IBMPIDhcpClient
99+
networkClient *instance.IBMPINetworkClient
99100
tgClient *transitgatewayapisv1.TransitGatewayApisV1
100101

101102
resourceGroupID string
@@ -235,12 +236,13 @@ func (o *ClusterUninstaller) destroyCluster() error {
235236
}, {
236237
{name: "Load Balancers", execute: o.destroyLoadBalancers},
237238
}, {
238-
{name: "Subnets", execute: o.destroySubnets},
239+
{name: "Cloud Subnets", execute: o.destroyCloudSubnets},
239240
}, {
240241
{name: "Public Gateways", execute: o.destroyPublicGateways},
241242
}, {
242243
{name: "DHCPs", execute: o.destroyDHCPNetworks},
243244
}, {
245+
{name: "Power Subnets", execute: o.destroyPowerSubnets},
244246
{name: "Images", execute: o.destroyImages},
245247
{name: "VPCs", execute: o.destroyVPCs},
246248
}, {
@@ -611,6 +613,11 @@ func (o *ClusterUninstaller) loadSDKServices() error {
611613
return fmt.Errorf("loadSDKServices: o.dhcpClient is nil")
612614
}
613615

616+
o.networkClient = instance.NewIBMPINetworkClient(context.Background(), o.piSession, o.ServiceGUID)
617+
if o.networkClient == nil {
618+
return fmt.Errorf("loadSDKServices: o.networkClient is nil")
619+
}
620+
614621
return nil
615622
}
616623

‎pkg/destroy/powervs/publicgateway.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ func (o *ClusterUninstaller) listAttachedSubnets(publicGatewayID string) (cloudR
3636
key: *subnet.ID,
3737
name: *subnet.Name,
3838
status: *subnet.Status,
39-
typeName: subnetTypeName,
39+
typeName: cloudSubnetTypeName,
4040
id: *subnet.ID,
4141
})
4242
}

0 commit comments

Comments
 (0)