Skip to content

Commit 08c7140

Browse files
tbaveliertbavelier
authored
[CONTINT-5028] Add permissions to collect Argo and Flux CRDs and use kustomize SA for e2e-tests (#2486)
* add timeout to e2e to see dumps instead of waiting CI job time out * fix RBAC for fluxcd * Do not use Helm chart SA and rely on kustomize one * clean up DDA before finishing test * Make cleanup more robust
1 parent 30df82e commit 08c7140

File tree

8 files changed

+156
-40
lines changed

8 files changed

+156
-40
lines changed

‎Makefile‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,10 @@ IMG_CHECK ?= gcr.io/datadoghq/operator-check:latest
5252

5353
# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
5454
ENVTEST_K8S_VERSION = 1.30
55+
# Default timeout for `go test` when running E2E tests.
56+
# (E2E provisioning can hang; having a finite timeout ensures we get goroutine dumps
57+
# instead of the CI job timing out with no actionable logs.)
58+
E2E_GO_TEST_TIMEOUT ?= 55m
5559

5660
# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
5761
ifeq (,$(shell go env GOBIN))
@@ -205,10 +209,10 @@ integration-tests: $(ENVTEST) ## Run integration tests with reconciler
205209
.PHONY: e2e-tests
206210
e2e-tests: ## Run E2E tests and destroy environment stacks after tests complete. To run locally, complete pre-reqs (see docs/how-to-contribute.md) and prepend command with `aws-vault exec sso-agent-sandbox-account-admin --`. E.g. `aws-vault exec sso-agent-sandbox-account-admin -- make e2e-tests`.
207211
@if [ -z "$(E2E_RUN_REGEX)" ]; then \
208-
KUBEBUILDER_ASSETS="$(ROOT)/bin/$(PLATFORM)/" go test -C test/e2e/ ./... -count=1 --tags=e2e -v -run TestAWSKindSuite -timeout 0s -coverprofile cover_e2e.out; \
212+
KUBEBUILDER_ASSETS="$(ROOT)/bin/$(PLATFORM)/" go test -C test/e2e/ ./... -count=1 --tags=e2e -v -run TestAWSKindSuite -timeout $(E2E_GO_TEST_TIMEOUT) -coverprofile cover_e2e.out; \
209213
else \
210214
echo "Running e2e test: $(E2E_RUN_REGEX)"; \
211-
KUBEBUILDER_ASSETS="$(ROOT)/bin/$(PLATFORM)/" go test -C test/e2e/ ./... -count=1 --tags=e2e -v -run $(E2E_RUN_REGEX) -timeout 0s -coverprofile cover_e2e.out; \
215+
KUBEBUILDER_ASSETS="$(ROOT)/bin/$(PLATFORM)/" go test -C test/e2e/ ./... -count=1 --tags=e2e -v -run $(E2E_RUN_REGEX) -timeout $(E2E_GO_TEST_TIMEOUT) -coverprofile cover_e2e.out; \
212216
fi
213217

214218
.PHONY: yaml-mapper-tests

‎config/rbac/role.yaml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,14 @@ rules:
128128
- get
129129
- list
130130
- watch
131+
- apiGroups:
132+
- argoproj.io
133+
resources:
134+
- applications
135+
- applicationsets
136+
verbs:
137+
- list
138+
- watch
131139
- apiGroups:
132140
- argoproj.io
133141
resources:
@@ -352,6 +360,13 @@ rules:
352360
- patch
353361
- update
354362
- watch
363+
- apiGroups:
364+
- kustomize.toolkit.fluxcd.io
365+
resources:
366+
- kustomizations
367+
verbs:
368+
- list
369+
- watch
355370
- apiGroups:
356371
- metrics.eks.amazonaws.com
357372
resources:
@@ -442,6 +457,18 @@ rules:
442457
- securitycontextconstraints
443458
verbs:
444459
- use
460+
- apiGroups:
461+
- source.toolkit.fluxcd.io
462+
resources:
463+
- buckets
464+
- externalartifacts
465+
- gitrepositories
466+
- helmcharts
467+
- helmrepositories
468+
- ocirepositories
469+
verbs:
470+
- list
471+
- watch
445472
- apiGroups:
446473
- storage.k8s.io
447474
resources:

‎internal/controller/datadogagent/feature/orchestratorexplorer/rbac.go‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,22 @@ func getRBACPolicyRules(logger logr.Logger, crs []string) []rbacv1.PolicyRule {
109109
},
110110
{
111111
APIGroups: []string{rbac.ArgoProjAPIGroup},
112-
Resources: []string{rbac.Rollout},
112+
Resources: []string{rbac.Rollout, rbac.Applications, rbac.Applicationsets},
113+
},
114+
{
115+
APIGroups: []string{rbac.FluxSourceToolkitAPIGroup},
116+
Resources: []string{
117+
rbac.Buckets,
118+
rbac.Helmcharts,
119+
rbac.Externalartifacts,
120+
rbac.Gitrepositories,
121+
rbac.Helmrepositories,
122+
rbac.Ocirepositories,
123+
},
124+
},
125+
{
126+
APIGroups: []string{rbac.FluxKustomizeToolkitAPIGroup},
127+
Resources: []string{rbac.Kustomizations},
113128
},
114129
{
115130
APIGroups: []string{rbac.KarpenterAPIGroup},

‎internal/controller/datadogagent_controller.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,15 @@ type DatadogAgentReconciler struct {
164164
// +kubebuilder:rbac:groups=discovery.k8s.io,resources=endpointslices,verbs=list;watch
165165
// +kubebuilder:rbac:groups=datadoghq.com,resources="*",verbs=list;watch
166166
// +kubebuilder:rbac:groups=argoproj.io,resources=rollouts,verbs=list;watch
167+
// +kubebuilder:rbac:groups=argoproj.io,resources=applications,verbs=list;watch
168+
// +kubebuilder:rbac:groups=argoproj.io,resources=applicationsets,verbs=list;watch
169+
// +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=buckets,verbs=list;watch
170+
// +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=externalartifacts,verbs=list;watch
171+
// +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=gitrepositories,verbs=list;watch
172+
// +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=helmcharts,verbs=list;watch
173+
// +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=helmrepositories,verbs=list;watch
174+
// +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=ocirepositories,verbs=list;watch
175+
// +kubebuilder:rbac:groups=kustomize.toolkit.fluxcd.io,resources=kustomizations,verbs=list;watch
167176
// +kubebuilder:rbac:groups=karpenter.sh,resources="*",verbs=get;list;watch;create;patch;update;delete
168177
// +kubebuilder:rbac:groups=karpenter.k8s.aws,resources="*",verbs=get;list;watch
169178
// +kubebuilder:rbac:groups=karpenter.azure.com,resources="*",verbs=list;watch

‎pkg/kubernetes/rbac/const.go‎

Lines changed: 38 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -11,32 +11,35 @@ const (
1111
Wildcard = "*"
1212

1313
// API Groups
14-
AdmissionAPIGroup = "admissionregistration.k8s.io"
15-
APIExtensionsAPIGroup = "apiextensions.k8s.io"
16-
AppsAPIGroup = "apps"
17-
ArgoProjAPIGroup = "argoproj.io"
18-
AuthorizationAPIGroup = "authorization.k8s.io"
19-
AutoscalingAPIGroup = "autoscaling"
20-
AutoscalingK8sIoAPIGroup = "autoscaling.k8s.io"
21-
BatchAPIGroup = "batch"
22-
CertificatesAPIGroup = "certificates.k8s.io"
23-
CoordinationAPIGroup = "coordination.k8s.io"
24-
CoreAPIGroup = ""
25-
DatadogAPIGroup = "datadoghq.com"
26-
DiscoveryAPIGroup = "discovery.k8s.io"
27-
ExtensionsAPIGroup = "extensions"
28-
ExternalMetricsAPIGroup = "external.metrics.k8s.io"
29-
GatewayAPIGroup = "gateway.networking.k8s.io"
30-
NetworkingAPIGroup = "networking.k8s.io"
31-
OpenShiftQuotaAPIGroup = "quota.openshift.io"
32-
PolicyAPIGroup = "policy"
33-
RbacAPIGroup = "rbac.authorization.k8s.io"
34-
RegistrationAPIGroup = "apiregistration.k8s.io"
35-
StorageAPIGroup = "storage.k8s.io"
36-
EKSMetricsAPIGroup = "metrics.eks.amazonaws.com"
37-
KarpenterAPIGroup = "karpenter.sh"
38-
KarpenterAWSAPIGroup = "karpenter.k8s.aws"
39-
KarpenterAzureAPIGroup = "karpenter.azure.com"
14+
AdmissionAPIGroup = "admissionregistration.k8s.io"
15+
APIExtensionsAPIGroup = "apiextensions.k8s.io"
16+
AppsAPIGroup = "apps"
17+
ArgoProjAPIGroup = "argoproj.io"
18+
// Flux CD API groups (Flux does not use a single "fluxcd.io" API group for these CRDs)
19+
FluxSourceToolkitAPIGroup = "source.toolkit.fluxcd.io"
20+
FluxKustomizeToolkitAPIGroup = "kustomize.toolkit.fluxcd.io"
21+
AuthorizationAPIGroup = "authorization.k8s.io"
22+
AutoscalingAPIGroup = "autoscaling"
23+
AutoscalingK8sIoAPIGroup = "autoscaling.k8s.io"
24+
BatchAPIGroup = "batch"
25+
CertificatesAPIGroup = "certificates.k8s.io"
26+
CoordinationAPIGroup = "coordination.k8s.io"
27+
CoreAPIGroup = ""
28+
DatadogAPIGroup = "datadoghq.com"
29+
DiscoveryAPIGroup = "discovery.k8s.io"
30+
ExtensionsAPIGroup = "extensions"
31+
ExternalMetricsAPIGroup = "external.metrics.k8s.io"
32+
GatewayAPIGroup = "gateway.networking.k8s.io"
33+
NetworkingAPIGroup = "networking.k8s.io"
34+
OpenShiftQuotaAPIGroup = "quota.openshift.io"
35+
PolicyAPIGroup = "policy"
36+
RbacAPIGroup = "rbac.authorization.k8s.io"
37+
RegistrationAPIGroup = "apiregistration.k8s.io"
38+
StorageAPIGroup = "storage.k8s.io"
39+
EKSMetricsAPIGroup = "metrics.eks.amazonaws.com"
40+
KarpenterAPIGroup = "karpenter.sh"
41+
KarpenterAWSAPIGroup = "karpenter.k8s.aws"
42+
KarpenterAzureAPIGroup = "karpenter.azure.com"
4043

4144
// Resources
4245

@@ -90,6 +93,15 @@ const (
9093
RoleBindingResource = "rolebindings"
9194
RoleResource = "roles"
9295
Rollout = "rollouts"
96+
Applications = "applications"
97+
Applicationsets = "applicationsets"
98+
Buckets = "buckets"
99+
Helmcharts = "helmcharts"
100+
Externalartifacts = "externalartifacts"
101+
Gitrepositories = "gitrepositories"
102+
Helmrepositories = "helmrepositories"
103+
Ocirepositories = "ocirepositories"
104+
Kustomizations = "kustomizations"
93105
SecretsResource = "secrets"
94106
ServiceAccountResource = "serviceaccounts"
95107
ServicesResource = "services"

‎test/e2e/tests/k8s_suite/k8s_suite_test.go‎

Lines changed: 43 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,16 @@ func (s *k8sSuite) TestGenericK8s() {
4444
defaultOperatorOpts := []operatorparams.Option{
4545
operatorparams.WithNamespace(common.NamespaceName),
4646
operatorparams.WithOperatorFullImagePath(common.OperatorImageName),
47-
operatorparams.WithHelmValues("installCRDs: false"),
47+
// RBAC/CRDs are installed via our e2e kustomize (`config/new-e2e`, namePrefix: datadog-operator-e2e-).
48+
// Ensure the Helm-installed operator uses the same ServiceAccount (and doesn't create its own RBAC),
49+
// otherwise it may run under a different SA (e.g. datadog-operator-linux) lacking new permissions.
50+
operatorparams.WithHelmValues(`installCRDs: false
51+
rbac:
52+
create: false
53+
serviceAccount:
54+
create: false
55+
name: datadog-operator-e2e-controller-manager
56+
`),
4857
}
4958

5059
defaultProvisionerOpts := []provisioners.KubernetesProvisionerOption{
@@ -57,6 +66,33 @@ func (s *k8sSuite) TestGenericK8s() {
5766
agentwithoperatorparams.WithNamespace(common.NamespaceName),
5867
}
5968

69+
// --- Suite-level cleanup (registered before any subtests run) ---
70+
//
71+
// We need to ensure the final env of the suite is left without a DatadogAgent before the
72+
// underlying Pulumi teardown happens; otherwise CRD deletion may race with DDA deletion.
73+
//
74+
// This runs ONCE, at the very end of the whole suite (not after each subtest).
75+
t := s.T()
76+
var lastTestName string
77+
updateEnv := func(testName string, opts []provisioners.KubernetesProvisionerOption) {
78+
lastTestName = testName
79+
s.UpdateEnv(provisioners.KubernetesProvisioner(opts...))
80+
}
81+
t.Cleanup(func() {
82+
if lastTestName == "" {
83+
return
84+
}
85+
86+
cleanupOpts := []provisioners.KubernetesProvisionerOption{
87+
provisioners.WithTestName(lastTestName),
88+
provisioners.WithK8sVersion(common.K8sVersion),
89+
provisioners.WithOperatorOptions(defaultOperatorOpts...),
90+
provisioners.WithoutDDA(),
91+
provisioners.WithLocal(s.local),
92+
}
93+
s.UpdateEnv(provisioners.KubernetesProvisioner(cleanupOpts...))
94+
})
95+
6096
s.T().Run("Verify Operator", func(t *testing.T) {
6197
s.Assert().EventuallyWithT(func(c *assert.CollectT) {
6298
utils.VerifyOperator(s.T(), c, common.NamespaceName, s.Env().KubernetesCluster.Client())
@@ -83,7 +119,7 @@ func (s *k8sSuite) TestGenericK8s() {
83119
provisioners.WithLocal(s.local),
84120
}
85121

86-
s.UpdateEnv(provisioners.KubernetesProvisioner(provisionerOptions...))
122+
updateEnv("e2e-operator-minimal-dda", provisionerOptions)
87123

88124
err = s.Env().FakeIntake.Client().FlushServerAndResetAggregators()
89125
s.Assert().NoError(err)
@@ -149,7 +185,7 @@ func (s *k8sSuite) TestGenericK8s() {
149185
provisioners.WithLocal(s.local),
150186
}
151187

152-
s.UpdateEnv(provisioners.KubernetesProvisioner(provisionerOptions...))
188+
updateEnv("e2e-operator-ksm-ccr", provisionerOptions)
153189

154190
err = s.Env().FakeIntake.Client().FlushServerAndResetAggregators()
155191
s.Assert().NoError(err)
@@ -190,7 +226,7 @@ func (s *k8sSuite) TestGenericK8s() {
190226
provisionerOptions = append(provisionerOptions, defaultProvisionerOpts...)
191227

192228
// Add nginx with annotations
193-
s.UpdateEnv(provisioners.KubernetesProvisioner(provisionerOptions...))
229+
updateEnv("e2e-operator-autodiscovery", provisionerOptions)
194230

195231
err = s.Env().FakeIntake.Client().FlushServerAndResetAggregators()
196232
s.Assert().NoError(err)
@@ -236,7 +272,7 @@ func (s *k8sSuite) TestGenericK8s() {
236272
provisioners.WithLocal(s.local),
237273
}
238274

239-
s.UpdateEnv(provisioners.KubernetesProvisioner(provisionerOptions...))
275+
updateEnv("e2e-operator-logs-collection", provisionerOptions)
240276

241277
// Verify logs collection on agent pod
242278
s.Assert().EventuallyWithTf(func(c *assert.CollectT) {
@@ -265,7 +301,7 @@ func (s *k8sSuite) TestGenericK8s() {
265301
provisioners.WithLocal(s.local),
266302
}
267303
withoutDDAProvisionerOptions = append(withoutDDAProvisionerOptions, defaultProvisionerOpts...)
268-
s.UpdateEnv(provisioners.KubernetesProvisioner(withoutDDAProvisionerOptions...))
304+
updateEnv("e2e-operator-apm", withoutDDAProvisionerOptions)
269305

270306
var apmAgentSelector = ",agent.datadoghq.com/name=datadog-agent-apm"
271307
ddaConfigPath, err := common.GetAbsPath(filepath.Join(common.ManifestsPath, "apm", "datadog-agent-apm.yaml"))
@@ -291,7 +327,7 @@ func (s *k8sSuite) TestGenericK8s() {
291327
ddaProvisionerOptions = append(ddaProvisionerOptions, defaultProvisionerOpts...)
292328

293329
// Deploy APM DatadogAgent and tracegen
294-
s.UpdateEnv(provisioners.KubernetesProvisioner(ddaProvisionerOptions...))
330+
updateEnv("e2e-operator-apm", ddaProvisionerOptions)
295331

296332
// Verify traces collection on agent pod
297333
s.EventuallyWithTf(func(c *assert.CollectT) {

‎test/e2e/tests/k8s_suite/kind_aws_test.go‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,13 @@ func TestAWSKindSuite(t *testing.T) {
2323
operatorOptions := []operatorparams.Option{
2424
operatorparams.WithNamespace(common.NamespaceName),
2525
operatorparams.WithOperatorFullImagePath(common.OperatorImageName),
26-
operatorparams.WithHelmValues("installCRDs: false"),
26+
operatorparams.WithHelmValues(`installCRDs: false
27+
rbac:
28+
create: false
29+
serviceAccount:
30+
create: false
31+
name: datadog-operator-e2e-controller-manager
32+
`),
2733
}
2834

2935
provisionerOptions := []provisioners.KubernetesProvisionerOption{

‎test/e2e/tests/k8s_suite/kind_local_test.go‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,13 @@ package k8ssuite
77

88
import (
99
"fmt"
10+
"strings"
11+
"testing"
12+
1013
"github.com/DataDog/datadog-agent/test/new-e2e/pkg/e2e"
1114
"github.com/DataDog/datadog-operator/test/e2e/common"
1215
"github.com/DataDog/datadog-operator/test/e2e/provisioners"
1316
"github.com/DataDog/test-infra-definitions/components/datadog/operatorparams"
14-
"strings"
15-
"testing"
1617
)
1718

1819
type localKindSuite struct {
@@ -29,7 +30,13 @@ func TestLocalKindSuite(t *testing.T) {
2930
operatorOptions := []operatorparams.Option{
3031
operatorparams.WithNamespace(common.NamespaceName),
3132
operatorparams.WithOperatorFullImagePath(common.OperatorImageName),
32-
operatorparams.WithHelmValues("installCRDs: false"),
33+
operatorparams.WithHelmValues(`installCRDs: false
34+
rbac:
35+
create: false
36+
serviceAccount:
37+
create: false
38+
name: datadog-operator-e2e-controller-manager
39+
`),
3340
}
3441

3542
provisionerOptions := []provisioners.KubernetesProvisionerOption{

0 commit comments

Comments
 (0)