feat: Reserve time to return preflight results before webhook timeout (#1235)

**What problem does this PR solve?**:
Assuming that preflight checks return within 2 seconds of context
cancellation, the preflight checks webhook will return before the API
server webhook timeout.

Some preflight checks already meet this criteria. I'm updating others to
meet it in
#1234
 
**Which issue(s) this PR fixes**:
Fixes #

**How Has This Been Tested?**:
<!--
Please describe the tests that you ran to verify your changes.
Provide output from the tests and any manual steps needed to replicate
the tests.
-->

**Special notes for your reviewer**:
<!--
Use this to provide any additional information to the reviewers.
This may include:
- Best way to review the PR.
- Where the author wants the most review attention on.
- etc.
-->

Author: dlipovetsky

pkg/webhook/preflight/doc.go

@@ -3,3 +3,5 @@
 package preflight
 
 // +kubebuilder:webhook:path=/preflight-v1beta1-cluster,mutating=false,failurePolicy=fail,groups="cluster.x-k8s.io",resources=clusters,verbs=create,versions=*,name=preflight.cluster.caren.nutanix.com,admissionReviewVersions=v1,sideEffects=None,timeoutSeconds=30
+
+// IMPORTANT Keep timeoutSeconds in sync with the `Timeout` constant defined in this package.

pkg/webhook/preflight/preflight.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"runtime/debug"
 	"sync"
+	"time"
 
 	admissionv1 "k8s.io/api/admission/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -19,6 +20,12 @@ import (
 	"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/webhook/preflight/skip"
 )
 
+const (
+	// Timeout is the duration, in seconds, that the preflight checks handler has to respond.
+	// IMPORTANT Keep in sync timeoutSeconds in the kubebuilder:webhook marker defined in this package.
+	Timeout = 30 * time.Second
+)
+
 type (
 	// Checker returns a set of checks that have been initialized with common dependencies,
 	// such as an infrastructure API client.
@@ -138,7 +145,12 @@ func (h *WebhookHandler) Handle(ctx context.Context, req admission.Request) admi
 		)
 	}
 
-	resultsOrderedByCheckerAndCheck := run(ctx, h.client, cluster, skipEvaluator, h.checkers)
+	// Reserve time for checks to handle context cancellation, so
+	// that we have time to summarize the results, and return a response.
+	checkTimeout := Timeout - 2*time.Second
+	checkCtx, checkCtxCancel := context.WithTimeout(ctx, checkTimeout)
+	resultsOrderedByCheckerAndCheck := run(checkCtx, h.client, cluster, skipEvaluator, h.checkers)
+	checkCtxCancel()
 
 	// Summarize the results.
 	resp := admission.Response{