Skip to content

Commit 560e0db

Browse files
sindhusri16joekr
sindhusri16
authored and
joekr
committed
feat: Add detailed work request errors when work-request is failed for both Loadbalancer and Machines (#378)
1 parent b563ee5 commit 560e0db

19 files changed

+600
-132
lines changed

‎OWNERS_ALIASES ‎ renamed to ‎OWNERS_ALIASES‎

File renamed without changes.

‎api/v1beta2/ocicluster_types.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -178,6 +178,12 @@ type ClientOverrides struct {
178178
// +optional
179179
// +nullable
180180
ContainerEngineClientUrl *string `json:"containerEngineClientUrl,omitempty"`
181+
182+
// WorkrequestClientUrl allows the default work request SDK client URL to be changed.
183+
//
184+
// +optional
185+
// +nullable
186+
WorkrequestClientUrl *string `json:"workrequestClientUrl,omitempty"`
181187
}
182188

183189
// GetConditions returns the list of conditions for an OCICluster API object.

‎api/v1beta2/zz_generated.deepcopy.go‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎cloud/ociutil/ociutil.go‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,13 @@ import (
2424

2525
lb "github.com/oracle/cluster-api-provider-oci/cloud/services/loadbalancer"
2626
nlb "github.com/oracle/cluster-api-provider-oci/cloud/services/networkloadbalancer"
27+
wrs "github.com/oracle/cluster-api-provider-oci/cloud/services/workrequests"
2728

2829
"github.com/oracle/oci-go-sdk/v65/common"
2930
"github.com/oracle/oci-go-sdk/v65/core"
3031
"github.com/oracle/oci-go-sdk/v65/loadbalancer"
3132
"github.com/oracle/oci-go-sdk/v65/networkloadbalancer"
33+
"github.com/oracle/oci-go-sdk/v65/workrequests"
3234

3335
"github.com/pkg/errors"
3436
"k8s.io/apimachinery/pkg/util/wait"
@@ -61,10 +63,25 @@ func IsNotFound(err error) bool {
6163
return ok && serviceErr.GetHTTPStatusCode() == http.StatusNotFound
6264
}
6365

66+
func FetchErrorsOnFailedWorkRequest(ctx context.Context, workRequestClient wrs.Client, workRequestId *string) (done bool, err error) {
67+
resp, err := workRequestClient.ListWorkRequestErrors(ctx, workrequests.ListWorkRequestErrorsRequest{
68+
WorkRequestId: workRequestId,
69+
})
70+
if err != nil {
71+
return false, errors.Wrapf(err, "Failed to fetch work-request-errors for failed workrequest: %s", *workRequestId)
72+
}
73+
final_err := errors.Errorf("WorkRequest %s failed", *workRequestId)
74+
for _, wr_err := range resp.Items {
75+
final_err = errors.Errorf("%s, %s", *wr_err.Message, final_err.Error())
76+
}
77+
return false, final_err
78+
}
79+
6480
// AwaitNLBWorkRequest waits for the LB work request to either succeed, fail. See k8s.io/apimachinery/pkg/util/wait
65-
func AwaitNLBWorkRequest(ctx context.Context, networkLoadBalancerClient nlb.NetworkLoadBalancerClient, workRequestId *string) (*networkloadbalancer.WorkRequest, error) {
81+
func AwaitNLBWorkRequest(ctx context.Context, networkLoadBalancerClient nlb.NetworkLoadBalancerClient, workRequestClient wrs.Client, workRequestId *string) (*networkloadbalancer.WorkRequest, error) {
6682
var wr *networkloadbalancer.WorkRequest
67-
err := wait.PollWithContext(ctx, WorkRequestPollInterval, WorkRequestTimeout, func(ctx context.Context) (done bool, err error) {
83+
immediate := true
84+
err := wait.PollUntilContextTimeout(ctx, WorkRequestPollInterval, WorkRequestTimeout, immediate, func(ctx context.Context) (done bool, err error) {
6885
twr, err := networkLoadBalancerClient.GetWorkRequest(ctx, networkloadbalancer.GetWorkRequestRequest{
6986
WorkRequestId: workRequestId,
7087
})
@@ -76,17 +93,18 @@ func AwaitNLBWorkRequest(ctx context.Context, networkLoadBalancerClient nlb.Netw
7693
wr = &twr.WorkRequest
7794
return true, nil
7895
case networkloadbalancer.OperationStatusFailed:
79-
return false, errors.Errorf("WorkRequest %s failed", *workRequestId)
96+
return FetchErrorsOnFailedWorkRequest(ctx, workRequestClient, workRequestId)
8097
}
8198
return false, nil
8299
})
83100
return wr, err
84101
}
85102

86103
// AwaitLBWorkRequest waits for the LBaaS work request to either succeed, fail. See k8s.io/apimachinery/pkg/util/wait
87-
func AwaitLBWorkRequest(ctx context.Context, loadBalancerClient lb.LoadBalancerClient, workRequestId *string) (*loadbalancer.WorkRequest, error) {
104+
func AwaitLBWorkRequest(ctx context.Context, loadBalancerClient lb.LoadBalancerClient, workRequestClient wrs.Client, workRequestId *string) (*loadbalancer.WorkRequest, error) {
88105
var wr *loadbalancer.WorkRequest
89-
err := wait.PollWithContext(ctx, WorkRequestPollInterval, WorkRequestTimeout, func(ctx context.Context) (done bool, err error) {
106+
immediate := true
107+
err := wait.PollUntilContextTimeout(ctx, WorkRequestPollInterval, WorkRequestTimeout, immediate, func(ctx context.Context) (done bool, err error) {
90108
twr, err := loadBalancerClient.GetWorkRequest(ctx, loadbalancer.GetWorkRequestRequest{
91109
WorkRequestId: workRequestId,
92110
})
@@ -98,7 +116,7 @@ func AwaitLBWorkRequest(ctx context.Context, loadBalancerClient lb.LoadBalancerC
98116
wr = &twr.WorkRequest
99117
return true, nil
100118
case loadbalancer.WorkRequestLifecycleStateFailed:
101-
return false, errors.Errorf("WorkRequest %s failed", *workRequestId)
119+
return FetchErrorsOnFailedWorkRequest(ctx, workRequestClient, workRequestId)
102120
}
103121
return false, nil
104122
})

‎cloud/scope/clients.go‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,15 @@ import (
3232
lb "github.com/oracle/cluster-api-provider-oci/cloud/services/loadbalancer"
3333
nlb "github.com/oracle/cluster-api-provider-oci/cloud/services/networkloadbalancer"
3434
"github.com/oracle/cluster-api-provider-oci/cloud/services/vcn"
35+
wr "github.com/oracle/cluster-api-provider-oci/cloud/services/workrequests"
3536
"github.com/oracle/cluster-api-provider-oci/version"
3637
"github.com/oracle/oci-go-sdk/v65/common"
3738
"github.com/oracle/oci-go-sdk/v65/containerengine"
3839
"github.com/oracle/oci-go-sdk/v65/core"
3940
"github.com/oracle/oci-go-sdk/v65/identity"
4041
"github.com/oracle/oci-go-sdk/v65/loadbalancer"
4142
"github.com/oracle/oci-go-sdk/v65/networkloadbalancer"
43+
"github.com/oracle/oci-go-sdk/v65/workrequests"
4244
"github.com/pkg/errors"
4345
"k8s.io/klog/v2/klogr"
4446
)
@@ -52,6 +54,7 @@ type OCIClients struct {
5254
LoadBalancerClient lb.LoadBalancerClient
5355
IdentityClient identityClient.Client
5456
ContainerEngineClient containerEngineClient.Client
57+
WorkRequestsClient wr.Client
5558
BaseClient base.BaseClient
5659
}
5760

@@ -161,11 +164,11 @@ func (c *ClientProvider) createClients(region string) (OCIClients, error) {
161164
if err != nil {
162165
return OCIClients{}, err
163166
}
164-
baseClient, err := c.createBaseClient(region, c.ociAuthConfigProvider, c.Logger)
167+
workrequestsClt, err := c.createWorkrequestsClient(region, c.ociAuthConfigProvider, c.Logger)
165168
if err != nil {
166169
return OCIClients{}, err
167170
}
168-
171+
baseClient, err := c.createBaseClient(region, c.ociAuthConfigProvider, c.Logger)
169172
if err != nil {
170173
return OCIClients{}, err
171174
}
@@ -178,6 +181,7 @@ func (c *ClientProvider) createClients(region string) (OCIClients, error) {
178181
ComputeClient: computeClient,
179182
ComputeManagementClient: computeManagementClient,
180183
ContainerEngineClient: containerEngineClt,
184+
WorkRequestsClient: workrequestsClt,
181185
BaseClient: baseClient,
182186
}, err
183187
}
@@ -308,6 +312,24 @@ func (c *ClientProvider) createContainerEngineClient(region string, ociAuthConfi
308312
return &containerEngineClt, nil
309313
}
310314

315+
func (c *ClientProvider) createWorkrequestsClient(region string, ociAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (*workrequests.WorkRequestClient, error) {
316+
workrequestsClt, err := workrequests.NewWorkRequestClientWithConfigurationProvider(ociAuthConfigProvider)
317+
if err != nil {
318+
logger.Error(err, "unable to create OCI WorkRequests client")
319+
return nil, err
320+
}
321+
workrequestsClt.SetRegion(region)
322+
dispatcher := workrequestsClt.HTTPClient
323+
workrequestsClt.HTTPClient = metrics.NewHttpRequestDispatcherWrapper(dispatcher, region)
324+
325+
if c.ociClientOverrides != nil && c.ociClientOverrides.WorkrequestClientUrl != nil {
326+
workrequestsClt.Host = *c.ociClientOverrides.WorkrequestClientUrl
327+
}
328+
workrequestsClt.Interceptor = setVersionHeader()
329+
330+
return &workrequestsClt, nil
331+
}
332+
311333
func (c *ClientProvider) createBaseClient(region string, ociAuthConfigProvider common.ConfigurationProvider, logger *logr.Logger) (base.BaseClient, error) {
312334
baseClient, err := base.NewBaseClient(ociAuthConfigProvider, logger)
313335
if err != nil {

‎cloud/scope/clients_mock.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131
"github.com/oracle/cluster-api-provider-oci/cloud/services/vcn"
3232
"github.com/oracle/oci-go-sdk/v65/loadbalancer"
3333
"github.com/oracle/oci-go-sdk/v65/networkloadbalancer"
34+
"github.com/oracle/oci-go-sdk/v65/workrequests"
3435
"k8s.io/klog/v2/klogr"
3536
)
3637

@@ -40,6 +41,7 @@ type MockOCIClients struct {
4041
NetworkLoadBalancerClient *networkloadbalancer.NetworkLoadBalancerClient
4142
LoadBalancerClient *loadbalancer.LoadBalancerClient
4243
IdentityClient identity.Client
44+
WorkRequestsClient *workrequests.WorkRequestClient
4345
}
4446

4547
var (
@@ -54,6 +56,7 @@ func MockNewClientProvider(mockClients MockOCIClients) (*ClientProvider, error)
5456
LoadBalancerClient: mockClients.LoadBalancerClient,
5557
IdentityClient: mockClients.IdentityClient,
5658
ComputeClient: mockClients.ComputeClient,
59+
WorkRequestsClient: mockClients.WorkRequestsClient,
5760
}}
5861

5962
authConfig, err := MockAuthConfig()

‎cloud/scope/clients_test.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ func TestClients_NewClientProviderWithClientOverrides(t *testing.T) {
6969
NetworkLoadBalancerClientUrl: common.String("NetworkLoadBalancerClientUrl"),
7070
IdentityClientUrl: common.String("IdentityClientUrl"),
7171
ContainerEngineClientUrl: common.String("ContainerEngineClientUrl"),
72+
WorkrequestClientUrl: common.String("WorkrequestClientUrl"),
7273
}
7374

7475
clientProvider, err := NewClientProvider(ClientProviderParams{
@@ -108,6 +109,7 @@ func TestClients_NewClientProviderWithMissingOverrides(t *testing.T) {
108109
LoadBalancerClientUrl: common.String("LoadBalancerClientUrl"),
109110
//NetworkLoadBalancerClientUrl is missing,
110111
//IdentityClientUrl is missing,
112+
//WorkrequestClientUrl is missing,
111113
ContainerEngineClientUrl: common.String("ContainerEngineClientUrl"),
112114
}
113115

‎cloud/scope/cluster.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
lb "github.com/oracle/cluster-api-provider-oci/cloud/services/loadbalancer"
3030
nlb "github.com/oracle/cluster-api-provider-oci/cloud/services/networkloadbalancer"
3131
"github.com/oracle/cluster-api-provider-oci/cloud/services/vcn"
32+
wr "github.com/oracle/cluster-api-provider-oci/cloud/services/workrequests"
3233
"github.com/oracle/oci-go-sdk/v65/common"
3334
"github.com/oracle/oci-go-sdk/v65/identity"
3435
"github.com/pkg/errors"
@@ -55,6 +56,7 @@ type ClusterScopeParams struct {
5556
NetworkLoadBalancerClient nlb.NetworkLoadBalancerClient
5657
LoadBalancerClient lb.LoadBalancerClient
5758
IdentityClient identityClient.Client
59+
WorkRequestClient wr.Client
5860
// RegionIdentifier Identifier as specified here https://docs.oracle.com/en-us/iaas/Content/General/Concepts/regions.htm
5961
RegionIdentifier string
6062
OCIAuthConfigProvider common.ConfigurationProvider
@@ -73,6 +75,7 @@ type ClusterScope struct {
7375
NetworkLoadBalancerClient nlb.NetworkLoadBalancerClient
7476
LoadBalancerClient lb.LoadBalancerClient
7577
IdentityClient identityClient.Client
78+
WorkRequestClient wr.Client
7679
// RegionIdentifier Identifier as specified here https://docs.oracle.com/en-us/iaas/Content/General/Concepts/regions.htm
7780
RegionIdentifier string
7881
ClientProvider *ClientProvider
@@ -104,6 +107,7 @@ func NewClusterScope(params ClusterScopeParams) (*ClusterScope, error) {
104107
NetworkLoadBalancerClient: params.NetworkLoadBalancerClient,
105108
LoadBalancerClient: params.LoadBalancerClient,
106109
IdentityClient: params.IdentityClient,
110+
WorkRequestClient: params.WorkRequestClient,
107111
RegionIdentifier: params.RegionIdentifier,
108112
ClientProvider: params.ClientProvider,
109113
OCIClusterAccessor: params.OCIClusterAccessor,

‎cloud/scope/drg_rpc_attachment_reconciler.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -253,7 +253,8 @@ func (s *ClusterScope) waitForRPCToBeProvisioned(ctx context.Context, rpc *core.
253253
return errors.New("invalid RPC lifecycle state")
254254
}
255255

256-
err := wait.PollWithContext(ctx, PollInterval, RequestTimeout, func(ctx context.Context) (done bool, err error) {
256+
immediate := true
257+
err := wait.PollUntilContextTimeout(ctx, PollInterval, RequestTimeout, immediate, func(ctx context.Context) (done bool, err error) {
257258
rpc, err := s.getRPC(ctx, rpc.Id, vcnClient)
258259
if err != nil {
259260
return true, err
@@ -271,7 +272,8 @@ func (s *ClusterScope) waitForRPCToBeProvisioned(ctx context.Context, rpc *core.
271272
}
272273

273274
func (s *ClusterScope) waitForRPCToBeDeleted(ctx context.Context, rpcId *string, vcnClient vcn.Client) error {
274-
err := wait.PollWithContext(ctx, PollInterval, RequestTimeout, func(ctx context.Context) (done bool, err error) {
275+
immediate := true
276+
err := wait.PollUntilContextTimeout(ctx, PollInterval, RequestTimeout, immediate, func(ctx context.Context) (done bool, err error) {
275277
rpc, err := s.getRPC(ctx, rpcId, vcnClient)
276278
if err != nil {
277279
if ociutil.IsNotFound(err) {

‎cloud/scope/load_balancer_reconciler.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ func (s *ClusterScope) DeleteApiServerLB(ctx context.Context) error {
8888
s.Logger.Error(err, "failed to delete apiserver lb")
8989
return errors.Wrap(err, "failed to delete apiserver lb")
9090
}
91-
_, err = ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, lbResponse.OpcWorkRequestId)
91+
_, err = ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, s.WorkRequestClient, lbResponse.OpcWorkRequestId)
9292
if err != nil {
9393
return errors.Wrap(err, "work request to delete lb failed")
9494
}
@@ -129,7 +129,7 @@ func (s *ClusterScope) UpdateLB(ctx context.Context, lb infrastructurev1beta2.Lo
129129
s.Logger.Error(err, "failed to reconcile the apiserver LB, failed to generate update lb workrequest")
130130
return errors.Wrap(err, "failed to reconcile the apiserver LB, failed to generate update lb workrequest")
131131
}
132-
_, err = ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, lbResponse.OpcWorkRequestId)
132+
_, err = ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, s.WorkRequestClient, lbResponse.OpcWorkRequestId)
133133
if err != nil {
134134
s.Logger.Error(err, "failed to reconcile the apiserver LB, failed to update lb")
135135
return errors.Wrap(err, "failed to reconcile the apiserver LB, failed to update lb")
@@ -202,7 +202,7 @@ func (s *ClusterScope) CreateLB(ctx context.Context, lb infrastructurev1beta2.Lo
202202
return nil, nil, errors.Wrap(err, "failed to create apiserver lb, failed to create work request")
203203
}
204204

205-
wr, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, lbResponse.OpcWorkRequestId)
205+
wr, err := ociutil.AwaitLBWorkRequest(ctx, s.LoadBalancerClient, s.WorkRequestClient, lbResponse.OpcWorkRequestId)
206206
if err != nil {
207207
return nil, nil, errors.Wrap(err, "awaiting load balancer")
208208
}

0 commit comments

Comments
 (0)