Skip to content

Commit a5bf6e6

Browse files
jpbetzjpbetz
committed
Replace time.Sleep with poll.wait in admission e2e tests
1 parent 363d1fe commit a5bf6e6

File tree

1 file changed

+170
-28
lines changed

1 file changed

+170
-28
lines changed

‎test/e2e/apimachinery/webhook.go‎

Lines changed: 170 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ var _ = SIGDescribe("AdmissionWebhook", func() {
9191

9292
// Make sure the namespace created for the test is labeled to be selected by the webhooks
9393
labelNamespace(f, f.Namespace.Name)
94+
createWebhookConfigurationReadyNamespace(f)
9495

9596
ginkgo.By("Setting up server cert")
9697
context = setupServerCert(namespaceName, serviceName)
@@ -150,7 +151,7 @@ var _ = SIGDescribe("AdmissionWebhook", func() {
150151
defer validatingWebhookCleanup()
151152
mutatingWebhookCleanup := registerMutatingWebhookForWebhookConfigurations(f, f.UniqueName+"blocking", context, servicePort)
152153
defer mutatingWebhookCleanup()
153-
testWebhooksForWebhookConfigurations(f, f.UniqueName, servicePort)
154+
testWebhooksForWebhookConfigurations(f, f.UniqueName, context, servicePort)
154155
})
155156

156157
ginkgo.It("Should mutate custom resource", func() {
@@ -683,12 +684,15 @@ func registerWebhook(f *framework.Framework, configName string, context *certCon
683684
// Server cannot talk to this webhook, so it always fails.
684685
// Because this webhook is configured fail-open, request should be admitted after the call fails.
685686
failOpenHook,
687+
688+
// Register a webhook that can be probed by marker requests to detect when the configuration is ready.
689+
newValidatingIsReadyWebhookFixture(f, context, servicePort),
686690
},
687691
})
688692
framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace)
689693

690-
// The webhook configuration is honored in 10s.
691-
time.Sleep(10 * time.Second)
694+
err = waitWebhookConfigurationReady(f)
695+
framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
692696

693697
return func() {
694698
client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(configName, nil)
@@ -733,12 +737,14 @@ func registerWebhookForAttachingPod(f *framework.Framework, configName string, c
733737
MatchLabels: map[string]string{f.UniqueName: "true"},
734738
},
735739
},
740+
// Register a webhook that can be probed by marker requests to detect when the configuration is ready.
741+
newValidatingIsReadyWebhookFixture(f, context, servicePort),
736742
},
737743
})
738744
framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace)
739745

740-
// The webhook configuration is honored in 10s.
741-
time.Sleep(10 * time.Second)
746+
err = waitWebhookConfigurationReady(f)
747+
framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
742748

743749
return func() {
744750
client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(configName, nil)
@@ -758,12 +764,14 @@ func registerMutatingWebhookForConfigMap(f *framework.Framework, configName stri
758764
Webhooks: []admissionregistrationv1.MutatingWebhook{
759765
newMutateConfigMapWebhookFixture(f, context, 1, servicePort),
760766
newMutateConfigMapWebhookFixture(f, context, 2, servicePort),
767+
// Register a webhook that can be probed by marker requests to detect when the configuration is ready.
768+
newMutatingIsReadyWebhookFixture(f, context, servicePort),
761769
},
762770
})
763771
framework.ExpectNoError(err, "registering mutating webhook config %s with namespace %s", configName, namespace)
764772

765-
// The webhook configuration is honored in 10s.
766-
time.Sleep(10 * time.Second)
773+
err = waitWebhookConfigurationReady(f)
774+
framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
767775
return func() { client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(configName, nil) }
768776
}
769777

@@ -821,12 +829,14 @@ func registerMutatingWebhookForPod(f *framework.Framework, configName string, co
821829
MatchLabels: map[string]string{f.UniqueName: "true"},
822830
},
823831
},
832+
// Register a webhook that can be probed by marker requests to detect when the configuration is ready.
833+
newMutatingIsReadyWebhookFixture(f, context, servicePort),
824834
},
825835
})
826836
framework.ExpectNoError(err, "registering mutating webhook config %s with namespace %s", configName, namespace)
827837

828-
// The webhook configuration is honored in 10s.
829-
time.Sleep(10 * time.Second)
838+
err = waitWebhookConfigurationReady(f)
839+
framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
830840

831841
return func() { client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(configName, nil) }
832842
}
@@ -1067,12 +1077,14 @@ func registerFailClosedWebhook(f *framework.Framework, configName string, contex
10671077
// Server cannot talk to this webhook, so it always fails.
10681078
// Because this webhook is configured fail-closed, request should be rejected after the call fails.
10691079
hook,
1080+
// Register a webhook that can be probed by marker requests to detect when the configuration is ready.
1081+
newValidatingIsReadyWebhookFixture(f, context, servicePort),
10701082
},
10711083
})
10721084
framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace)
10731085

1074-
// The webhook configuration is honored in 10s.
1075-
time.Sleep(10 * time.Second)
1086+
err = waitWebhookConfigurationReady(f)
1087+
framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
10761088
return func() {
10771089
f.ClientSet.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(configName, nil)
10781090
}
@@ -1151,12 +1163,14 @@ func registerValidatingWebhookForWebhookConfigurations(f *framework.Framework, c
11511163
MatchLabels: map[string]string{f.UniqueName: "true"},
11521164
},
11531165
},
1166+
// Register a webhook that can be probed by marker requests to detect when the configuration is ready.
1167+
newValidatingIsReadyWebhookFixture(f, context, servicePort),
11541168
},
11551169
})
11561170
framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace)
11571171

1158-
// The webhook configuration is honored in 10s.
1159-
time.Sleep(10 * time.Second)
1172+
err = waitWebhookConfigurationReady(f)
1173+
framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
11601174
return func() {
11611175
err := client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(configName, nil)
11621176
framework.ExpectNoError(err, "deleting webhook config %s with namespace %s", configName, namespace)
@@ -1210,12 +1224,14 @@ func registerMutatingWebhookForWebhookConfigurations(f *framework.Framework, con
12101224
MatchLabels: map[string]string{f.UniqueName: "true"},
12111225
},
12121226
},
1227+
// Register a webhook that can be probed by marker requests to detect when the configuration is ready.
1228+
newMutatingIsReadyWebhookFixture(f, context, servicePort),
12131229
},
12141230
})
12151231
framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace)
12161232

1217-
// The webhook configuration is honored in 10s.
1218-
time.Sleep(10 * time.Second)
1233+
err = waitWebhookConfigurationReady(f)
1234+
framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
12191235
return func() {
12201236
err := client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(configName, nil)
12211237
framework.ExpectNoError(err, "deleting webhook config %s with namespace %s", configName, namespace)
@@ -1225,7 +1241,7 @@ func registerMutatingWebhookForWebhookConfigurations(f *framework.Framework, con
12251241
// This test assumes that the deletion-rejecting webhook defined in
12261242
// registerValidatingWebhookForWebhookConfigurations and the webhook-config-mutating
12271243
// webhook defined in registerMutatingWebhookForWebhookConfigurations already exist.
1228-
func testWebhooksForWebhookConfigurations(f *framework.Framework, configName string, servicePort int32) {
1244+
func testWebhooksForWebhookConfigurations(f *framework.Framework, configName string, context *certContext, servicePort int32) {
12291245
var err error
12301246
client := f.ClientSet
12311247
ginkgo.By("Creating a dummy validating-webhook-configuration object")
@@ -1271,15 +1287,17 @@ func testWebhooksForWebhookConfigurations(f *framework.Framework, configName str
12711287
MatchLabels: map[string]string{f.UniqueName: "true"},
12721288
},
12731289
},
1290+
// Register a webhook that can be probed by marker requests to detect when the configuration is ready.
1291+
newValidatingIsReadyWebhookFixture(f, context, servicePort),
12741292
},
12751293
})
12761294
framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace)
12771295
if mutatedValidatingWebhookConfiguration.ObjectMeta.Labels != nil && mutatedValidatingWebhookConfiguration.ObjectMeta.Labels[addedLabelKey] == addedLabelValue {
12781296
e2elog.Failf("expected %s not to be mutated by mutating webhooks but it was", configName)
12791297
}
12801298

1281-
// The webhook configuration is honored in 10s.
1282-
time.Sleep(10 * time.Second)
1299+
err = waitWebhookConfigurationReady(f)
1300+
framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
12831301

12841302
ginkgo.By("Deleting the validating-webhook-configuration, which should be possible to remove")
12851303

@@ -1325,15 +1343,17 @@ func testWebhooksForWebhookConfigurations(f *framework.Framework, configName str
13251343
MatchLabels: map[string]string{f.UniqueName: "true"},
13261344
},
13271345
},
1346+
// Register a webhook that can be probed by marker requests to detect when the configuration is ready.
1347+
newMutatingIsReadyWebhookFixture(f, context, servicePort),
13281348
},
13291349
})
13301350
framework.ExpectNoError(err, "registering webhook config %s with namespace %s", configName, namespace)
13311351
if mutatedMutatingWebhookConfiguration.ObjectMeta.Labels != nil && mutatedMutatingWebhookConfiguration.ObjectMeta.Labels[addedLabelKey] == addedLabelValue {
13321352
e2elog.Failf("expected %s not to be mutated by mutating webhooks but it was", configName)
13331353
}
13341354

1335-
// The webhook configuration is honored in 10s.
1336-
time.Sleep(10 * time.Second)
1355+
err = waitWebhookConfigurationReady(f)
1356+
framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
13371357

13381358
ginkgo.By("Deleting the mutating-webhook-configuration, which should be possible to remove")
13391359

@@ -1542,12 +1562,14 @@ func registerWebhookForCustomResource(f *framework.Framework, configName string,
15421562
MatchLabels: map[string]string{f.UniqueName: "true"},
15431563
},
15441564
},
1565+
// Register a webhook that can be probed by marker requests to detect when the configuration is ready.
1566+
newValidatingIsReadyWebhookFixture(f, context, servicePort),
15451567
},
15461568
})
15471569
framework.ExpectNoError(err, "registering custom resource webhook config %s with namespace %s", configName, namespace)
15481570

1549-
// The webhook configuration is honored in 10s.
1550-
time.Sleep(10 * time.Second)
1571+
err = waitWebhookConfigurationReady(f)
1572+
framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
15511573
return func() {
15521574
client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(configName, nil)
15531575
}
@@ -1617,12 +1639,14 @@ func registerMutatingWebhookForCustomResource(f *framework.Framework, configName
16171639
MatchLabels: map[string]string{f.UniqueName: "true"},
16181640
},
16191641
},
1642+
// Register a webhook that can be probed by marker requests to detect when the configuration is ready.
1643+
newMutatingIsReadyWebhookFixture(f, context, servicePort),
16201644
},
16211645
})
16221646
framework.ExpectNoError(err, "registering custom resource webhook config %s with namespace %s", configName, namespace)
16231647

1624-
// The webhook configuration is honored in 10s.
1625-
time.Sleep(10 * time.Second)
1648+
err = waitWebhookConfigurationReady(f)
1649+
framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
16261650

16271651
return func() { client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(configName, nil) }
16281652
}
@@ -1819,12 +1843,14 @@ func registerValidatingWebhookForCRD(f *framework.Framework, configName string,
18191843
MatchLabels: map[string]string{f.UniqueName: "true"},
18201844
},
18211845
},
1846+
// Register a webhook that can be probed by marker requests to detect when the configuration is ready.
1847+
newValidatingIsReadyWebhookFixture(f, context, servicePort),
18221848
},
18231849
})
18241850
framework.ExpectNoError(err, "registering crd webhook config %s with namespace %s", configName, namespace)
18251851

1826-
// The webhook configuration is honored in 10s.
1827-
time.Sleep(10 * time.Second)
1852+
err = waitWebhookConfigurationReady(f)
1853+
framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
18281854
return func() {
18291855
client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(configName, nil)
18301856
}
@@ -1943,12 +1969,14 @@ func registerSlowWebhook(f *framework.Framework, configName string, context *cer
19431969
SideEffects: &sideEffectsNone,
19441970
AdmissionReviewVersions: []string{"v1", "v1beta1"},
19451971
},
1972+
// Register a webhook that can be probed by marker requests to detect when the configuration is ready.
1973+
newValidatingIsReadyWebhookFixture(f, context, servicePort),
19461974
},
19471975
})
19481976
framework.ExpectNoError(err, "registering slow webhook config %s with namespace %s", configName, namespace)
19491977

1950-
// The webhook configuration is honored in 10s.
1951-
time.Sleep(10 * time.Second)
1978+
err = waitWebhookConfigurationReady(f)
1979+
framework.ExpectNoError(err, "waiting for webhook configuration to be ready")
19521980

19531981
return func() {
19541982
client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Delete(configName, nil)
@@ -2148,3 +2176,117 @@ func newMutateConfigMapWebhookFixture(f *framework.Framework, context *certConte
21482176
},
21492177
}
21502178
}
2179+
2180+
// createWebhookConfigurationReadyNamespace creates a separate namespace for webhook configuration ready markers to
2181+
// prevent cross-talk with webhook configurations being tested.
2182+
func createWebhookConfigurationReadyNamespace(f *framework.Framework) {
2183+
ns, err := f.ClientSet.CoreV1().Namespaces().Create(&v1.Namespace{
2184+
ObjectMeta: metav1.ObjectMeta{
2185+
Name: f.Namespace.Name + "-markers",
2186+
Labels: map[string]string{f.UniqueName + "-markers": "true"},
2187+
},
2188+
})
2189+
framework.ExpectNoError(err, "creating namespace for webhook configuration ready markers")
2190+
f.AddNamespacesToDelete(ns)
2191+
}
2192+
2193+
// waitWebhookConfigurationReady sends "marker" requests until a webhook configuration is ready.
2194+
// A webhook created with newValidatingIsReadyWebhookFixture or newMutatingIsReadyWebhookFixture should first be added to
2195+
// the webhook configuration.
2196+
func waitWebhookConfigurationReady(f *framework.Framework) error {
2197+
cmClient := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name + "-markers")
2198+
return wait.PollImmediate(100*time.Millisecond, 30*time.Second, func() (bool, error) {
2199+
marker := &v1.ConfigMap{
2200+
ObjectMeta: metav1.ObjectMeta{
2201+
Name: string(uuid.NewUUID()),
2202+
Labels: map[string]string{
2203+
f.UniqueName: "true",
2204+
},
2205+
},
2206+
}
2207+
_, err := cmClient.Create(marker)
2208+
if err != nil {
2209+
// The always-deny webhook does not provide a reason, so check for the error string we expect
2210+
if strings.Contains(err.Error(), "denied") {
2211+
return true, nil
2212+
}
2213+
return false, err
2214+
}
2215+
// best effort cleanup of markers that are no longer needed
2216+
_ = cmClient.Delete(marker.GetName(), nil)
2217+
framework.Logf("Waiting for webhook configuration to be ready...")
2218+
return false, nil
2219+
})
2220+
}
2221+
2222+
// newValidatingIsReadyWebhookFixture creates a validating webhook that can be added to a webhook configuration and then probed
2223+
// with "marker" requests via waitWebhookConfigurationReady to wait for a webhook configuration to be ready.
2224+
func newValidatingIsReadyWebhookFixture(f *framework.Framework, context *certContext, servicePort int32) admissionregistrationv1.ValidatingWebhook {
2225+
sideEffectsNone := admissionregistrationv1.SideEffectClassNone
2226+
return admissionregistrationv1.ValidatingWebhook{
2227+
Name: "validating-is-webhook-configuration-ready.k8s.io",
2228+
Rules: []admissionregistrationv1.RuleWithOperations{{
2229+
Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create},
2230+
Rule: admissionregistrationv1.Rule{
2231+
APIGroups: []string{""},
2232+
APIVersions: []string{"v1"},
2233+
Resources: []string{"configmaps"},
2234+
},
2235+
}},
2236+
ClientConfig: admissionregistrationv1.WebhookClientConfig{
2237+
Service: &admissionregistrationv1.ServiceReference{
2238+
Namespace: f.Namespace.Name,
2239+
Name: serviceName,
2240+
Path: strPtr("/always-deny"),
2241+
Port: pointer.Int32Ptr(servicePort),
2242+
},
2243+
CABundle: context.signingCert,
2244+
},
2245+
SideEffects: &sideEffectsNone,
2246+
AdmissionReviewVersions: []string{"v1", "v1beta1"},
2247+
// Scope the webhook to just the markers namespace
2248+
NamespaceSelector: &metav1.LabelSelector{
2249+
MatchLabels: map[string]string{f.UniqueName + "-markers": "true"},
2250+
},
2251+
// appease createValidatingWebhookConfiguration isolation requirements
2252+
ObjectSelector: &metav1.LabelSelector{
2253+
MatchLabels: map[string]string{f.UniqueName: "true"},
2254+
},
2255+
}
2256+
}
2257+
2258+
// newMutatingIsReadyWebhookFixture creates a mutating webhook that can be added to a webhook configuration and then probed
2259+
// with "marker" requests via waitWebhookConfigurationReady to wait for a webhook configuration to be ready.
2260+
func newMutatingIsReadyWebhookFixture(f *framework.Framework, context *certContext, servicePort int32) admissionregistrationv1.MutatingWebhook {
2261+
sideEffectsNone := admissionregistrationv1.SideEffectClassNone
2262+
return admissionregistrationv1.MutatingWebhook{
2263+
Name: "mutating-is-webhook-configuration-ready.k8s.io",
2264+
Rules: []admissionregistrationv1.RuleWithOperations{{
2265+
Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create},
2266+
Rule: admissionregistrationv1.Rule{
2267+
APIGroups: []string{""},
2268+
APIVersions: []string{"v1"},
2269+
Resources: []string{"configmaps"},
2270+
},
2271+
}},
2272+
ClientConfig: admissionregistrationv1.WebhookClientConfig{
2273+
Service: &admissionregistrationv1.ServiceReference{
2274+
Namespace: f.Namespace.Name,
2275+
Name: serviceName,
2276+
Path: strPtr("/always-deny"),
2277+
Port: pointer.Int32Ptr(servicePort),
2278+
},
2279+
CABundle: context.signingCert,
2280+
},
2281+
SideEffects: &sideEffectsNone,
2282+
AdmissionReviewVersions: []string{"v1", "v1beta1"},
2283+
// Scope the webhook to just the markers namespace
2284+
NamespaceSelector: &metav1.LabelSelector{
2285+
MatchLabels: map[string]string{f.UniqueName + "-markers": "true"},
2286+
},
2287+
// appease createMutatingWebhookConfiguration isolation requirements
2288+
ObjectSelector: &metav1.LabelSelector{
2289+
MatchLabels: map[string]string{f.UniqueName: "true"},
2290+
},
2291+
}
2292+
}

0 commit comments

Comments
 (0)