diff --git a/test/e2e/parallel/nstemplatetier_test.go b/test/e2e/parallel/nstemplatetier_test.go index 3319efb76..377f2fbf6 100644 --- a/test/e2e/parallel/nstemplatetier_test.go +++ b/test/e2e/parallel/nstemplatetier_test.go @@ -13,6 +13,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "github.com/gofrs/uuid" + "k8s.io/client-go/kubernetes/scheme" toolchainv1alpha1 "github.com/codeready-toolchain/api/api/v1alpha1" "github.com/codeready-toolchain/toolchain-common/pkg/states" @@ -21,8 +22,6 @@ import ( . "github.com/codeready-toolchain/toolchain-e2e/testsupport/space" "github.com/codeready-toolchain/toolchain-e2e/testsupport/tiers" "github.com/codeready-toolchain/toolchain-e2e/testsupport/wait" - apiwait "k8s.io/apimachinery/pkg/util/wait" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/labels" @@ -378,6 +377,101 @@ func TestTierTemplateRevision(t *testing.T) { require.NoError(t, err) // for the tiertemplaterevisions to be created the tiertemplates need to have template objects populated // we add the RawExtension objects to the TemplateObjects field + crq := getTestCRQ("600") + rawTemplateObjects := []runtime.RawExtension{{Object: &crq}} + updateTierTemplateObjects := func(template *toolchainv1alpha1.TierTemplate) error { + template.Spec.TemplateObjects = rawTemplateObjects + return nil + } + // for simplicity, we add the CRQ to all types of templates (both cluster scope and namespace scoped), + // even if the CRQ is cluster scoped. + // WARNING: thus THIS NSTemplateTier should NOT be used to provision a user!!! + customTier := tiers.CreateCustomNSTemplateTier(t, hostAwait, "ttr", baseTier, + tiers.WithNamespaceResources(t, baseTier, updateTierTemplateObjects), + tiers.WithClusterResources(t, baseTier, updateTierTemplateObjects), + tiers.WithSpaceRoles(t, baseTier, updateTierTemplateObjects), + tiers.WithParameter("DEPLOYMENT_QUOTA", "60")) + // when + // we verify that TierTemplateRevision CRs were created, since all the tiertemplates now have templateObjects field populated + tier, err := hostAwait.WaitForNSTemplateTierAndCheckTemplates(t, "ttr", + wait.HasStatusTierTemplateRevisions(tiers.GetTemplateRefs(t, hostAwait, "ttr").Flatten())) + require.NoError(t, err) + customTier.NSTemplateTier = tier + + // then + // check the expected total number of ttr matches, + // we IDEALLY expect one TTR per each tiertemplate to be created (clusterresource, namespace and spacerole), thus a total of 3 TTRs ideally. + // But since the creation of a TTR could be very quick and could trigger another reconcile of the NSTemplateTier before the status is actually updated with the reference, + // this might generate some copies of the TTRs. This is not a problem in production since the cleanup mechanism of TTRs will remove the extra ones but could cause some flakiness with the test, + // thus we assert the number of TTRs doesn't exceed the double of the expected number. + // TODO check for exact match or remove the *2 and check for not empty revisions list, once we implement the cleanup controller + ttrs, err := hostAwait.WaitForTTRs(t, customTier.Name, wait.LessOrEqual(len(tiers.GetTemplateRefs(t, hostAwait, "ttr").Flatten())*2)) + require.NoError(t, err) + + t.Run("update of tiertemplate should trigger creation of new TTR", func(t *testing.T) { + // given + // that the tiertemplates and nstemlpatetier are provisioned from the parent test + ttrToBeModified, found := customTier.Status.Revisions[customTier.Spec.ClusterResources.TemplateRef] + require.True(t, found) + // check that it has the crq before updating it + checkThatTTRContainsCRQ(t, ttrToBeModified, ttrs, crq) + + // when + // we update one tiertemplate + // let's reduce the pod count + updatedCRQ := getTestCRQ("100") + _, err = wait.For(t, hostAwait.Awaitility, &toolchainv1alpha1.TierTemplate{}). + Update(customTier.Spec.ClusterResources.TemplateRef, hostAwait.Namespace, func(tiertemplate *toolchainv1alpha1.TierTemplate) { + tiertemplate.Spec.TemplateObjects = []runtime.RawExtension{{Object: &updatedCRQ}} + }) + require.NoError(t, err) + + // then + // a new TTR was created + // TODO check for exact match or remove the +1 once we implement the cleanup controller + updatedTTRs, err := hostAwait.WaitForTTRs(t, customTier.Name, wait.GreaterOrEqual(len(ttrs)+1)) + require.NoError(t, err) + // get the updated nstemplatetier + updatedCustomTier, err := hostAwait.WaitForNSTemplateTier(t, customTier.Name) + newTTR, found := updatedCustomTier.Status.Revisions[updatedCustomTier.Spec.ClusterResources.TemplateRef] + require.True(t, found) + // check that it has the updated crq + checkThatTTRContainsCRQ(t, newTTR, updatedTTRs, updatedCRQ) + + t.Run("update of the NSTemplateTier parameters should trigger creation of new TTR", func(t *testing.T) { + // given + // that the TierTemplates and NSTemplateTier are provisioned from the parent test + // and they have the initial parameter value for deployment quota + checkThatTTRsHaveParameter(t, customTier, updatedTTRs, toolchainv1alpha1.Parameter{ + Name: "DEPLOYMENT_QUOTA", + Value: "60", + }) + + // when + // we increase the parameter for the deployment quota + customTier = tiers.UpdateCustomNSTemplateTier(t, hostAwait, customTier, tiers.WithParameter("DEPLOYMENT_QUOTA", "100")) + require.NoError(t, err) + + // then + // an additional TTR will be created + // TODO check for exact match or remove the +1 once we implement the cleanup controller + ttrsWithNewParams, err := hostAwait.WaitForTTRs(t, customTier.Name, wait.GreaterOrEqual(len(updatedTTRs)+1)) + require.NoError(t, err) + // retrieve new tier once the ttrs were created and the revision field updated + customTier.NSTemplateTier, err = hostAwait.WaitForNSTemplateTier(t, customTier.Name) + require.NoError(t, err) + // and the parameter is updated in all the ttrs + checkThatTTRsHaveParameter(t, customTier, ttrsWithNewParams, toolchainv1alpha1.Parameter{ + Name: "DEPLOYMENT_QUOTA", + Value: "100", + }) + }) + + }) + +} + +func getTestCRQ(podsCount string) unstructured.Unstructured { crq := unstructured.Unstructured{Object: map[string]interface{}{ "kind": "ClusterResourceQuota", "metadata": map[string]interface{}{ @@ -385,69 +479,23 @@ func TestTierTemplateRevision(t *testing.T) { }, "spec": map[string]interface{}{ "quota": map[string]interface{}{ - "hard": map[string]string{ + "hard": map[string]interface{}{ "count/deploymentconfigs.apps": "{{.DEPLOYMENT_QUOTA}}", "count/deployments.apps": "{{.DEPLOYMENT_QUOTA}}", - "count/pods": "600", + "count/pods": podsCount, }, }, "selector": map[string]interface{}{ - "annotations": map[string]string{}, + "annotations": map[string]interface{}{}, "labels": map[string]interface{}{ - "matchLabels": map[string]string{ + "matchLabels": map[string]interface{}{ "toolchain.dev.openshift.com/space": "{{.SPACE_NAME}}", }, }, }, }, }} - rawTemplateObjects := []runtime.RawExtension{{Object: &crq}} - updateTierTemplateObjects := func(template *toolchainv1alpha1.TierTemplate) error { - template.Spec.TemplateObjects = rawTemplateObjects - return nil - } - // for simplicity, we add the CRQ to all types of templates (both cluster scope and namespace scoped), - // even if the CRQ is cluster scoped. - // WARNING: thus THIS NSTemplateTier should NOT be sued to provision a user!!! - tiers.CreateCustomNSTemplateTier(t, hostAwait, "ttr", baseTier, - tiers.WithNamespaceResources(t, baseTier, updateTierTemplateObjects), - tiers.WithClusterResources(t, baseTier, updateTierTemplateObjects), - tiers.WithSpaceRoles(t, baseTier, updateTierTemplateObjects), - tiers.WithParameter("DEPLOYMENT_QUOTA", "60")) - // when - // we verify the counters in the status.history for 'tierUsingTierTemplateRevisions' tier - // and verify that TierTemplateRevision CRs were created, since all the tiertemplates now have templateObjects field populated - customTier, err := hostAwait.WaitForNSTemplateTierAndCheckTemplates(t, "ttr", - wait.HasStatusTierTemplateRevisions(tiers.GetTemplateRefs(t, hostAwait, "ttr").Flatten())) - require.NoError(t, err) - - // then - // check the expected total number of ttr matches - err = apiwait.PollUntilContextTimeout(context.TODO(), hostAwait.RetryInterval, hostAwait.Timeout, true, func(ctx context.Context) (done bool, err error) { - objs := &toolchainv1alpha1.TierTemplateRevisionList{} - if err := hostAwait.Client.List(ctx, objs, client.InNamespace(hostAwait.Namespace)); err != nil { - return false, err - } - // we IDEALLY expect one TTR per each tiertemplate to be created (clusterresource, namespace and spacerole), thus a total of 3 TTRs ideally. - // But since the creation of a TTR could be very quick and could trigger another reconcile of the NSTemplateTier before the status is actually updated with the reference, - // this might generate some copies of the TTRs. This is not a problem in production since the cleanup mechanism of TTRs will remove the extra ones but could cause some flakiness with the test, - // thus we assert the number of TTRs doesn't exceed the double of the expected number. - assert.LessOrEqual(t, len(objs.Items), len(tiers.GetTemplateRefs(t, hostAwait, "ttr").Flatten())*2) - // we check that the TTR content has the parameters replaced with values from the NSTemplateTier - for _, obj := range objs.Items { - // the object should have all the variables still there since this one will be replaced when provisioning the Space - assert.Contains(t, string(obj.Spec.TemplateObjects[0].Raw), ".SPACE_NAME") - assert.Contains(t, string(obj.Spec.TemplateObjects[0].Raw), ".DEPLOYMENT_QUOTA") - // the parameter is copied from the NSTemplateTier - assert.NotNil(t, obj.Spec.Parameters) - assert.NotNil(t, customTier.Spec.Parameters) - // we only expect the static parameter DEPLOYMENT_QUOTA to be copied from the tier to the TTR. - // the SPACE_NAME is not a parameter, but a dynamic variable which will be evaluated when provisioning a namespace for the user. - assert.ElementsMatch(t, customTier.Spec.Parameters, obj.Spec.Parameters) - } - return true, nil - }) - require.NoError(t, err) + return crq } func withClusterRoleBindings(t *testing.T, otherTier *toolchainv1alpha1.NSTemplateTier, feature string) tiers.CustomNSTemplateTierModifier { @@ -498,3 +546,29 @@ const ( } ` ) + +// checkThatTTRContainsCRQ verifies if a given ttr from the list contains the CRQ in the templateObjects field +func checkThatTTRContainsCRQ(t *testing.T, ttrName string, ttrs []toolchainv1alpha1.TierTemplateRevision, crq unstructured.Unstructured) { + for _, ttr := range ttrs { + if ttr.Name == ttrName { + assert.NotEmpty(t, ttr.Spec.TemplateObjects) + unstructuredObj := &unstructured.Unstructured{} + _, _, err := scheme.Codecs.UniversalDeserializer().Decode(ttr.Spec.TemplateObjects[0].Raw, nil, unstructuredObj) + require.NoError(t, err) + assert.Equal(t, &crq, unstructuredObj) + return + } + } + require.FailNowf(t, "Unable to find a TTR with required crq", "ttr:%s CRQ:%+v", ttrName, crq) +} + +// checkThatTTRsHaveParameter verifies that ttrs from the list have the required parameter +func checkThatTTRsHaveParameter(t *testing.T, tier *tiers.CustomNSTemplateTier, ttrs []toolchainv1alpha1.TierTemplateRevision, parameters toolchainv1alpha1.Parameter) { + for _, ttr := range ttrs { + // if the ttr is still in the revisions field we check that it contains the required parameters + if ttrNameInRev, ttrFound := tier.Status.Revisions[ttr.GetLabels()[toolchainv1alpha1.TemplateRefLabelKey]]; ttrFound && ttrNameInRev == ttr.Name { + assert.Contains(t, ttr.Spec.Parameters, parameters, "Unable to find required parameters:%+v in the TTR:%s", parameters, ttr.Name) + return + } + } +} diff --git a/testsupport/tiers/tier_setup.go b/testsupport/tiers/tier_setup.go index a1fe4a34a..e5d1c47f6 100644 --- a/testsupport/tiers/tier_setup.go +++ b/testsupport/tiers/tier_setup.go @@ -86,6 +86,15 @@ func WithParameter(name, value string) CustomNSTemplateTierModifier { if tier.Spec.Parameters == nil { tier.Spec.Parameters = []toolchainv1alpha1.Parameter{} } + + for i, param := range tier.Spec.Parameters { + if param.Name == name { + // if the param already exists, let's set the new value + tier.Spec.Parameters[i].Value = value + return nil + } + } + // if it's a new parameter, let's append it to the existing ones tier.Spec.Parameters = append(tier.Spec.Parameters, toolchainv1alpha1.Parameter{ Name: name, @@ -143,7 +152,7 @@ func UpdateCustomNSTemplateTier(t *testing.T, hostAwait *wait.HostAwaitility, ti err := modify(hostAwait, tier) require.NoError(t, err) } - _, err = wait.For(t, hostAwait.Awaitility, &toolchainv1alpha1.NSTemplateTier{}). + tier.NSTemplateTier, err = wait.For(t, hostAwait.Awaitility, &toolchainv1alpha1.NSTemplateTier{}). Update(tier.NSTemplateTier.Name, hostAwait.Namespace, func(nstt *toolchainv1alpha1.NSTemplateTier) { nstt.Spec = tier.NSTemplateTier.Spec }) diff --git a/testsupport/wait/host.go b/testsupport/wait/host.go index 83f125fef..4baa4f6a5 100644 --- a/testsupport/wait/host.go +++ b/testsupport/wait/host.go @@ -994,6 +994,100 @@ func (a *HostAwaitility) WaitForTierTemplate(t *testing.T, name string) (*toolch return tierTemplate, err } +// TierTemplateRevisionWaitCriterion a struct to compare with an expected TierTemplateRevision +type TierTemplateRevisionWaitCriterion struct { + Match func([]toolchainv1alpha1.TierTemplateRevision) bool + Diff func([]toolchainv1alpha1.TierTemplateRevision) string +} + +func matchTierTemplateRevisionWaitCriterion(actual []toolchainv1alpha1.TierTemplateRevision, criteria ...TierTemplateRevisionWaitCriterion) bool { + for _, c := range criteria { + // if at least one criteria does not match, keep waiting + if !c.Match(actual) { + return false + } + } + return true +} + +func (a *HostAwaitility) printTierTemplateRevisionWaitCriterionDiffs(t *testing.T, actual []toolchainv1alpha1.TierTemplateRevision, tierName string, criteria ...TierTemplateRevisionWaitCriterion) { + buf := &strings.Builder{} + if len(actual) == 0 { + buf.WriteString("no ttrs found\n") + } else { + buf.WriteString("failed to find ttrs with matching criteria:\n") + buf.WriteString("actual:\n") + for _, obj := range actual { + y, _ := StringifyObject(&obj) // nolint:gosec + buf.Write(y) + } + buf.WriteString("\n----\n") + buf.WriteString("diffs:\n") + for _, c := range criteria { + if !c.Match(actual) { + buf.WriteString(c.Diff(actual)) + buf.WriteString("\n") + } + } + } + opts := client.MatchingLabels(map[string]string{ + toolchainv1alpha1.TierLabelKey: tierName, + }) + // include also all TierTemplateRevisions for the given tier, to help troubleshooting + a.listAndPrint(t, "TierTemplateRevisions", a.Namespace, &toolchainv1alpha1.TierTemplateRevisionList{}, opts) + // include also all TierTemplate for the given tiertemplate revisions, to help troubleshooting + for _, ttr := range actual { + a.GetAndPrint(t, "TierTemplate", a.Namespace, ttr.GetLabels()[toolchainv1alpha1.TemplateRefLabelKey], &toolchainv1alpha1.TierTemplate{}) + } + + t.Log(buf.String()) +} + +// GreaterOrEqual checks if the number of TTRs is greater or equal than the expected one +func GreaterOrEqual(count int) TierTemplateRevisionWaitCriterion { + return TierTemplateRevisionWaitCriterion{ + Match: func(actual []toolchainv1alpha1.TierTemplateRevision) bool { + return len(actual) >= count + }, + Diff: func(actual []toolchainv1alpha1.TierTemplateRevision) string { + return fmt.Sprintf("number of ttrs %d is not greater or equal than %d \n", len(actual), count) + }, + } +} + +// LessOrEqual checks if the number of TTRs is less or equal than the expected one +func LessOrEqual(count int) TierTemplateRevisionWaitCriterion { + return TierTemplateRevisionWaitCriterion{ + Match: func(actual []toolchainv1alpha1.TierTemplateRevision) bool { + return len(actual) <= count + }, + Diff: func(actual []toolchainv1alpha1.TierTemplateRevision) string { + return fmt.Sprintf("number of ttrs %d is not less or equal than %d \n", len(actual), count) + }, + } +} + +func (a *HostAwaitility) WaitForTTRs(t *testing.T, tierName string, criteria ...TierTemplateRevisionWaitCriterion) ([]toolchainv1alpha1.TierTemplateRevision, error) { + t.Logf("waiting for ttrs to match criteria for tier '%s'", tierName) + var ttrs []toolchainv1alpha1.TierTemplateRevision + err := wait.PollUntilContextTimeout(context.TODO(), a.RetryInterval, a.Timeout, true, func(ctx context.Context) (done bool, err error) { + objs := &toolchainv1alpha1.TierTemplateRevisionList{} + if err := a.Client.List(ctx, objs, client.InNamespace(a.Namespace), client.MatchingLabels{toolchainv1alpha1.TierLabelKey: tierName}); err != nil { + return false, err + } + if len(objs.Items) == 0 { + return false, nil + } + ttrs = objs.Items + return matchTierTemplateRevisionWaitCriterion(ttrs, criteria...), nil + }) + // no match found, print the diffs + if err != nil { + a.printTierTemplateRevisionWaitCriterionDiffs(t, ttrs, tierName, criteria...) + } + return ttrs, err +} + // NSTemplateTierWaitCriterion a struct to compare with an expected NSTemplateTier type NSTemplateTierWaitCriterion struct { Match func(*toolchainv1alpha1.NSTemplateTier) bool