Skip to content
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
122 changes: 72 additions & 50 deletions test/e2e/parallel/nstemplatetier_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,6 @@ import (
. "github.com/codeready-toolchain/toolchain-e2e/testsupport/space"
"github.com/codeready-toolchain/toolchain-e2e/testsupport/tiers"
"github.com/codeready-toolchain/toolchain-e2e/testsupport/wait"
apiwait "k8s.io/apimachinery/pkg/util/wait"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"k8s.io/apimachinery/pkg/labels"
Expand Down Expand Up @@ -378,6 +376,76 @@ func TestTierTemplateRevision(t *testing.T) {
require.NoError(t, err)
// for the tiertemplaterevisions to be created the tiertemplates need to have template objects populated
// we add the RawExtension objects to the TemplateObjects field
crq := getTestCRQ("600")
rawTemplateObjects := []runtime.RawExtension{{Object: &crq}}
updateTierTemplateObjects := func(template *toolchainv1alpha1.TierTemplate) error {
template.Spec.TemplateObjects = rawTemplateObjects
return nil
}
// for simplicity, we add the CRQ to all types of templates (both cluster scope and namespace scoped),
// even if the CRQ is cluster scoped.
// WARNING: thus THIS NSTemplateTier should NOT be used to provision a user!!!
customTier := tiers.CreateCustomNSTemplateTier(t, hostAwait, "ttr", baseTier,
tiers.WithNamespaceResources(t, baseTier, updateTierTemplateObjects),
tiers.WithClusterResources(t, baseTier, updateTierTemplateObjects),
tiers.WithSpaceRoles(t, baseTier, updateTierTemplateObjects),
tiers.WithParameter("DEPLOYMENT_QUOTA", "60"))
// when
// we verify the counters in the status.history for 'tierUsingTierTemplateRevisions' tier
// and verify that TierTemplateRevision CRs were created, since all the tiertemplates now have templateObjects field populated
_, err = hostAwait.WaitForNSTemplateTierAndCheckTemplates(t, "ttr",
wait.HasStatusTierTemplateRevisions(tiers.GetTemplateRefs(t, hostAwait, "ttr").Flatten()))
require.NoError(t, err)

// then
// check the expected total number of ttr matches
// we IDEALLY expect one TTR per each tiertemplate to be created (clusterresource, namespace and spacerole), thus a total of 3 TTRs ideally.
// But since the creation of a TTR could be very quick and could trigger another reconcile of the NSTemplateTier before the status is actually updated with the reference,
// this might generate some copies of the TTRs. This is not a problem in production since the cleanup mechanism of TTRs will remove the extra ones but could cause some flakiness with the test,
// thus we assert the number of TTRs doesn't exceed the double of the expected number.
ttrs, err := hostAwait.WaitForTTRs(t, customTier.Name, wait.LessOrEqual(len(tiers.GetTemplateRefs(t, hostAwait, "ttr").Flatten())*2))
require.NoError(t, err)

t.Run("update of tiertemplate should trigger creation of new TTR", func(t *testing.T) {
// given
// that the tiertemplates and nstemlpatetier are provisioned from the parent test

// when
// we update one tiertemplate
_, err = wait.For(t, hostAwait.Awaitility, &toolchainv1alpha1.TierTemplate{}).
Update(customTier.Spec.ClusterResources.TemplateRef, hostAwait.Namespace, func(tiertemplate *toolchainv1alpha1.TierTemplate) {
// let's reduce the pod count
updatedCRQ := getTestCRQ("100")
tiertemplate.Spec.TemplateObjects = []runtime.RawExtension{{Object: &updatedCRQ}}
})
require.NoError(t, err)

// then
// a new TTR was created
updatedTTRs, err := hostAwait.WaitForTTRs(t, customTier.Name, wait.GreaterOrEqual(len(ttrs)+1))
require.NoError(t, err)

t.Run("update of the NSTemplateTier parameters should trigger creation of new TTR", func(t *testing.T) {
// given
// that the TierTemplates and NSTemplateTier are provisioned from the parent test

// when
// we update a parameter in the NSTemplateTier
// by increasing the deployment quota
customTier = tiers.UpdateCustomNSTemplateTier(t, hostAwait, customTier, tiers.WithParameter("DEPLOYMENT_QUOTA", "100"))
require.NoError(t, err)

// then
// an additional TTR was created
_, err := hostAwait.WaitForTTRs(t, customTier.Name, wait.GreaterOrEqual(len(updatedTTRs)+1))
require.NoError(t, err)
})

})

}

func getTestCRQ(podsCount string) unstructured.Unstructured {
crq := unstructured.Unstructured{Object: map[string]interface{}{
"kind": "ClusterResourceQuota",
"metadata": map[string]interface{}{
Expand All @@ -388,7 +456,7 @@ func TestTierTemplateRevision(t *testing.T) {
"hard": map[string]string{
"count/deploymentconfigs.apps": "{{.DEPLOYMENT_QUOTA}}",
"count/deployments.apps": "{{.DEPLOYMENT_QUOTA}}",
"count/pods": "600",
"count/pods": podsCount,
},
},
"selector": map[string]interface{}{
Expand All @@ -401,53 +469,7 @@ func TestTierTemplateRevision(t *testing.T) {
},
},
}}
rawTemplateObjects := []runtime.RawExtension{{Object: &crq}}
updateTierTemplateObjects := func(template *toolchainv1alpha1.TierTemplate) error {
template.Spec.TemplateObjects = rawTemplateObjects
return nil
}
// for simplicity, we add the CRQ to all types of templates (both cluster scope and namespace scoped),
// even if the CRQ is cluster scoped.
// WARNING: thus THIS NSTemplateTier should NOT be sued to provision a user!!!
tiers.CreateCustomNSTemplateTier(t, hostAwait, "ttr", baseTier,
tiers.WithNamespaceResources(t, baseTier, updateTierTemplateObjects),
tiers.WithClusterResources(t, baseTier, updateTierTemplateObjects),
tiers.WithSpaceRoles(t, baseTier, updateTierTemplateObjects),
tiers.WithParameter("DEPLOYMENT_QUOTA", "60"))
// when
// we verify the counters in the status.history for 'tierUsingTierTemplateRevisions' tier
// and verify that TierTemplateRevision CRs were created, since all the tiertemplates now have templateObjects field populated
customTier, err := hostAwait.WaitForNSTemplateTierAndCheckTemplates(t, "ttr",
wait.HasStatusTierTemplateRevisions(tiers.GetTemplateRefs(t, hostAwait, "ttr").Flatten()))
require.NoError(t, err)

// then
// check the expected total number of ttr matches
err = apiwait.PollUntilContextTimeout(context.TODO(), hostAwait.RetryInterval, hostAwait.Timeout, true, func(ctx context.Context) (done bool, err error) {
objs := &toolchainv1alpha1.TierTemplateRevisionList{}
if err := hostAwait.Client.List(ctx, objs, client.InNamespace(hostAwait.Namespace)); err != nil {
return false, err
}
// we IDEALLY expect one TTR per each tiertemplate to be created (clusterresource, namespace and spacerole), thus a total of 3 TTRs ideally.
// But since the creation of a TTR could be very quick and could trigger another reconcile of the NSTemplateTier before the status is actually updated with the reference,
// this might generate some copies of the TTRs. This is not a problem in production since the cleanup mechanism of TTRs will remove the extra ones but could cause some flakiness with the test,
// thus we assert the number of TTRs doesn't exceed the double of the expected number.
assert.LessOrEqual(t, len(objs.Items), len(tiers.GetTemplateRefs(t, hostAwait, "ttr").Flatten())*2)
// we check that the TTR content has the parameters replaced with values from the NSTemplateTier
for _, obj := range objs.Items {
// the object should have all the variables still there since this one will be replaced when provisioning the Space
assert.Contains(t, string(obj.Spec.TemplateObjects[0].Raw), ".SPACE_NAME")
assert.Contains(t, string(obj.Spec.TemplateObjects[0].Raw), ".DEPLOYMENT_QUOTA")
// the parameter is copied from the NSTemplateTier
assert.NotNil(t, obj.Spec.Parameters)
assert.NotNil(t, customTier.Spec.Parameters)
// we only expect the static parameter DEPLOYMENT_QUOTA to be copied from the tier to the TTR.
// the SPACE_NAME is not a parameter, but a dynamic variable which will be evaluated when provisioning a namespace for the user.
assert.ElementsMatch(t, customTier.Spec.Parameters, obj.Spec.Parameters)
}
return true, nil
})
require.NoError(t, err)
return crq
}

func withClusterRoleBindings(t *testing.T, otherTier *toolchainv1alpha1.NSTemplateTier, feature string) tiers.CustomNSTemplateTierModifier {
Expand Down
28 changes: 22 additions & 6 deletions testsupport/tiers/tier_setup.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,12 +86,28 @@ func WithParameter(name, value string) CustomNSTemplateTierModifier {
if tier.Spec.Parameters == nil {
tier.Spec.Parameters = []toolchainv1alpha1.Parameter{}
}
tier.Spec.Parameters = append(tier.Spec.Parameters,
toolchainv1alpha1.Parameter{
Name: name,
Value: value,
},
)

newParams := make([]toolchainv1alpha1.Parameter, 0, len(tier.Spec.Parameters))
// search for the param, in case it already exists, we need only to change its value
found := false
for _, param := range tier.Spec.Parameters {
if param.Name == name {
// if the param already exists, let's set the new value
param.Value = value
found = true
}
newParams = append(newParams, param)
}
// if it's a new parameter, let's append it to the existing ones
if !found {
newParams = append(newParams,
toolchainv1alpha1.Parameter{
Name: name,
Value: value,
},
)
}
tier.Spec.Parameters = newParams
return nil
}
}
Expand Down
94 changes: 94 additions & 0 deletions testsupport/wait/host.go
Original file line number Diff line number Diff line change
Expand Up @@ -994,6 +994,100 @@ func (a *HostAwaitility) WaitForTierTemplate(t *testing.T, name string) (*toolch
return tierTemplate, err
}

// TierTemplateRevisionWaitCriterion a struct to compare with an expected TierTemplateRevision
type TierTemplateRevisionWaitCriterion struct {
Match func([]toolchainv1alpha1.TierTemplateRevision) bool
Diff func([]toolchainv1alpha1.TierTemplateRevision) string
}

func matchTierTemplateRevisionWaitCriterion(actual []toolchainv1alpha1.TierTemplateRevision, criteria ...TierTemplateRevisionWaitCriterion) bool {
for _, c := range criteria {
// if at least one criteria does not match, keep waiting
if !c.Match(actual) {
return false
}
}
return true
}

func (a *HostAwaitility) printTierTemplateRevisionWaitCriterionDiffs(t *testing.T, actual []toolchainv1alpha1.TierTemplateRevision, tierName string, criteria ...TierTemplateRevisionWaitCriterion) {
buf := &strings.Builder{}
if len(actual) == 0 {
buf.WriteString("no ttrs found\n")
} else {
buf.WriteString("failed to find ttrs with matching criteria:\n")
buf.WriteString("actual:\n")
for _, obj := range actual {
y, _ := StringifyObject(&obj) // nolint:gosec
buf.Write(y)
}
buf.WriteString("\n----\n")
buf.WriteString("diffs:\n")
for _, c := range criteria {
if !c.Match(actual) {
buf.WriteString(c.Diff(actual))
buf.WriteString("\n")
}
}
}
opts := client.MatchingLabels(map[string]string{
toolchainv1alpha1.TierLabelKey: tierName,
})
// include also all TierTemplateRevisions for the given tier, to help troubleshooting
a.listAndPrint(t, "TierTemplateRevisions", a.Namespace, &toolchainv1alpha1.TierTemplateRevisionList{}, opts)
// include also all TierTemplate for the given tiertemplate revisions, to help troubleshooting
for _, ttr := range actual {
a.GetAndPrint(t, "TierTemplate", a.Namespace, ttr.GetLabels()[toolchainv1alpha1.TemplateRefLabelKey], &toolchainv1alpha1.TierTemplate{})
}

t.Log(buf.String())
}

// GreaterOrEqual checks if the number of TTRs is greater or equal than the expected one
func GreaterOrEqual(count int) TierTemplateRevisionWaitCriterion {
return TierTemplateRevisionWaitCriterion{
Match: func(actual []toolchainv1alpha1.TierTemplateRevision) bool {
return len(actual) >= count
},
Diff: func(actual []toolchainv1alpha1.TierTemplateRevision) string {
return fmt.Sprintf("number of ttrs %d is not greater or equal than %d \n", len(actual), count)
},
}
}

// LessOrEqual checks if the number of TTRs is less or equal than the expected one
func LessOrEqual(count int) TierTemplateRevisionWaitCriterion {
return TierTemplateRevisionWaitCriterion{
Match: func(actual []toolchainv1alpha1.TierTemplateRevision) bool {
return len(actual) <= count
},
Diff: func(actual []toolchainv1alpha1.TierTemplateRevision) string {
return fmt.Sprintf("number of ttrs %d is not less or equal than %d \n", len(actual), count)
},
}
}
Comment on lines +1046 to +1068
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

btw, these could be generic functions available as part of the new wait.For API
but that's for later, not for this PR 😉

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah, not huge expert and fan of generic code, but I agree it might make sense.

Actually there's also a lot of margin for improvement in the assertions of the TTRs as well. We can do those later if that makes sense.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As mentioned during the sync call, I found a way we can use the standard assertion functions
#1073 (comment)


func (a *HostAwaitility) WaitForTTRs(t *testing.T, tierName string, criteria ...TierTemplateRevisionWaitCriterion) ([]toolchainv1alpha1.TierTemplateRevision, error) {
t.Logf("waiting for ttrs to match criteria for tier '%s'", tierName)
var ttrs []toolchainv1alpha1.TierTemplateRevision
err := wait.PollUntilContextTimeout(context.TODO(), a.RetryInterval, a.Timeout, true, func(ctx context.Context) (done bool, err error) {
objs := &toolchainv1alpha1.TierTemplateRevisionList{}
if err := a.Client.List(ctx, objs, client.InNamespace(a.Namespace), client.MatchingLabels{toolchainv1alpha1.TierLabelKey: tierName}); err != nil {
return false, err
}
if len(objs.Items) == 0 {
return false, nil
}
ttrs = objs.Items
return matchTierTemplateRevisionWaitCriterion(ttrs, criteria...), nil
})
// no match found, print the diffs
if err != nil {
a.printTierTemplateRevisionWaitCriterionDiffs(t, ttrs, tierName, criteria...)
}
return ttrs, err
}

// NSTemplateTierWaitCriterion a struct to compare with an expected NSTemplateTier
type NSTemplateTierWaitCriterion struct {
Match func(*toolchainv1alpha1.NSTemplateTier) bool
Expand Down
Loading