Skip to content

Commit 9489c01

Browse files
kcp-ci-botkcp-ci-bot
authored
Merge pull request #2 from kcp-dev/e2e-tests
Add end-to-end testing
2 parents 38e798b + da83a3c commit 9489c01

File tree

15 files changed

+1030
-60
lines changed

15 files changed

+1030
-60
lines changed

‎.wwhrd.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,3 +25,8 @@ allowlist:
2525
- BSD-2-Clause-FreeBSD
2626
- BSD-3-Clause
2727
- ISC
28+
29+
exceptions:
30+
- github.com/hashicorp/golang-lru/v2 # MPL 2.0
31+
- github.com/hashicorp/golang-lru/v2/internal # MPL 2.0
32+
- github.com/hashicorp/golang-lru/v2/simplelru # MPL 2.0

‎cmd/init-agent/main.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -103,9 +103,9 @@ func run(ctx context.Context, log *zap.SugaredLogger, opts *Options) error {
103103
return fmt.Errorf("failed to setup source factory: %w", err)
104104
}
105105

106-
// clusterApplier controls how the manifests of an init source are applied in
106+
// manifestApplier controls how the manifests of an init source are applied in
107107
// the target workspace
108-
clusterApplier := manifest.NewClusterApplier(clusterClient)
108+
manifestApplier := manifest.NewApplier()
109109

110110
// create the ctrl-runtime manager
111111
mgr, err := setupManager(ctx, cfg, opts)
@@ -119,7 +119,7 @@ func run(ctx context.Context, log *zap.SugaredLogger, opts *Options) error {
119119
// wrap this controller creation in a closure to prevent giving all the initcontroller
120120
// dependencies to the targetcontroller
121121
newInitController := func(remoteManager mcmanager.Manager, targetProvider initcontroller.InitTargetProvider, initializer kcpcorev1alpha1.LogicalClusterInitializer) error {
122-
return initcontroller.Create(remoteManager, targetProvider, sourceFactory, clusterApplier, initializer, log, numInitWorkers)
122+
return initcontroller.Create(remoteManager, targetProvider, sourceFactory, manifestApplier, initializer, log, numInitWorkers)
123123
}
124124

125125
if err := targetcontroller.Add(ctx, mgr, log, opts.InitTargetSelector, clusterClient, newInitController); err != nil {

‎go.mod‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ replace github.com/kcp-dev/init-agent/sdk => ./sdk
66

77
require (
88
github.com/Masterminds/sprig/v3 v3.3.0
9+
github.com/go-logr/logr v1.4.3
910
github.com/go-logr/zapr v1.3.0
1011
github.com/kcp-dev/init-agent/sdk v0.0.0-00010101000000-000000000000
1112
github.com/kcp-dev/logicalcluster/v3 v3.0.5
@@ -14,6 +15,7 @@ require (
1415
github.com/spf13/pflag v1.0.10
1516
go.uber.org/zap v1.27.1
1617
k8s.io/api v0.34.2
18+
k8s.io/apiextensions-apiserver v0.34.2
1719
k8s.io/apimachinery v0.34.2
1820
k8s.io/client-go v0.34.2
1921
k8s.io/utils v0.0.0-20250604170112-4c0f3b243397
@@ -32,7 +34,6 @@ require (
3234
github.com/evanphx/json-patch/v5 v5.9.11 // indirect
3335
github.com/fsnotify/fsnotify v1.9.0 // indirect
3436
github.com/fxamacker/cbor/v2 v2.9.0 // indirect
35-
github.com/go-logr/logr v1.4.3 // indirect
3637
github.com/go-openapi/jsonpointer v0.22.0 // indirect
3738
github.com/go-openapi/jsonreference v0.21.1 // indirect
3839
github.com/go-openapi/swag v0.24.1 // indirect
@@ -52,6 +53,7 @@ require (
5253
github.com/google/gnostic-models v0.7.0 // indirect
5354
github.com/google/go-cmp v0.7.0 // indirect
5455
github.com/google/uuid v1.6.0 // indirect
56+
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
5557
github.com/huandu/xstrings v1.5.0 // indirect
5658
github.com/josharian/intern v1.0.0 // indirect
5759
github.com/json-iterator/go v1.1.12 // indirect
@@ -92,7 +94,6 @@ require (
9294
gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect
9395
gopkg.in/inf.v0 v0.9.1 // indirect
9496
gopkg.in/yaml.v3 v3.0.1 // indirect
95-
k8s.io/apiextensions-apiserver v0.34.2 // indirect
9697
k8s.io/klog/v2 v2.130.1 // indirect
9798
k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect
9899
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect

‎go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,8 @@ github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J
8484
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA=
8585
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
8686
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
87+
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
88+
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
8789
github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI=
8890
github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
8991
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=

‎hack/ci/run-e2e-tests.sh‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright 2026 The kcp Authors.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -euo pipefail
18+
source hack/lib.sh
19+
20+
# have a place to store things
21+
if [ -z "${ARTIFACTS:-}" ]; then
22+
ARTIFACTS=.e2e/artifacts
23+
mkdir -p "$ARTIFACTS"
24+
fi
25+
26+
echodate "Build artifacts will be placed in $ARTIFACTS."
27+
export ARTIFACTS="$(realpath "$ARTIFACTS")"
28+
29+
# build the agent, we will start it many times during the tests
30+
echodate "Building the init-agent…"
31+
make build
32+
33+
# start a shared kcp process
34+
KCP="$(UGET_PRINT_PATH=relative make --no-print-directory install-kcp)"
35+
36+
KCP_ROOT_DIRECTORY=.kcp.e2e
37+
KCP_LOGFILE="$ARTIFACTS/kcp.log"
38+
KCP_TOKENFILE=hack/ci/testdata/e2e-kcp.tokens
39+
40+
echodate "Starting kcp…"
41+
rm -rf "$KCP_ROOT_DIRECTORY" "$KCP_LOGFILE"
42+
"$KCP" start \
43+
-v4 \
44+
--token-auth-file "$KCP_TOKENFILE" \
45+
--root-directory "$KCP_ROOT_DIRECTORY" 1>"$KCP_LOGFILE" 2>&1 &
46+
47+
stop_kcp() {
48+
echodate "Stopping kcp processes (set \$KEEP_KCP=true to not do this)…"
49+
pkill -e kcp
50+
}
51+
52+
if [[ -v KEEP_KCP ]] && $KEEP_KCP; then
53+
echodate "\$KEEP_KCP is set, will not stop kcp once the script is finished."
54+
else
55+
append_trap stop_kcp EXIT
56+
fi
57+
58+
# make the token available to the Go tests
59+
export KCP_AGENT_TOKEN="$(grep e2e "$KCP_TOKENFILE" | cut -f1 -d,)"
60+
61+
# Wait for kcp to be ready; this env name is also hardcoded in the Go tests.
62+
export KCP_KUBECONFIG="$KCP_ROOT_DIRECTORY/admin.kubeconfig"
63+
64+
# the tenancy API becomes available pretty late during startup, so it's a good readiness check
65+
KUBECTL="$(UGET_PRINT_PATH=relative make --no-print-directory install-kubectl)"
66+
if ! retry_linear 3 20 "$KUBECTL" --kubeconfig "$KCP_KUBECONFIG" get workspaces; then
67+
echodate "kcp never became ready."
68+
exit 1
69+
fi
70+
71+
# makes it easier to reference these files from various _test.go files.
72+
export ROOT_DIRECTORY="$(realpath .)"
73+
export KCP_KUBECONFIG="$(realpath "$KCP_KUBECONFIG")"
74+
export AGENT_BINARY="$(realpath _build/init-agent)"
75+
76+
# time to run the tests
77+
echodate "Running e2e tests…"
78+
WHAT="${WHAT:-./test/e2e/...}"
79+
(set -x; go test -tags e2e -timeout 2h -v $WHAT)
80+
81+
echodate "Done. :-)"

‎hack/ci/testdata/e2e-kcp.tokens‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
topphemmelig,init-agent-e2e,1111-2222-3333-4444,"init-agents"

‎hack/run-e2e-tests.sh‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright 2026 The kcp Authors.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -euo pipefail
18+
19+
cd "$(dirname $0)/.."
20+
21+
source hack/lib.sh
22+
23+
export ARTIFACTS=.e2e
24+
25+
rm -rf "$ARTIFACTS"
26+
mkdir -p "$ARTIFACTS"
27+
28+
KCP_ROOT_DIRECTORY="$ARTIFACTS/kcp"
29+
KCP_LOGFILE="$ARTIFACTS/kcp.log"
30+
KCP_TOKENFILE=hack/ci/testdata/e2e-kcp.tokens
31+
KCP_PID=0
32+
33+
echodate "Starting kcp…"
34+
KCP="$(UGET_PRINT_PATH=relative make --no-print-directory install-kcp)"
35+
36+
rm -rf "$KCP_ROOT_DIRECTORY" "$KCP_LOGFILE"
37+
"$KCP" start \
38+
-v4 \
39+
--token-auth-file "$KCP_TOKENFILE" \
40+
--root-directory "$KCP_ROOT_DIRECTORY" 1>"$KCP_LOGFILE" 2>&1 &
41+
KCP_PID=$!
42+
43+
stop_kcp() {
44+
echodate "Stopping kcp (set \$KEEP_KCP=true to not do this)…"
45+
kill -TERM $KCP_PID
46+
wait $KCP_PID
47+
}
48+
49+
if [[ -v KEEP_KCP ]] && $KEEP_KCP; then
50+
echodate "\$KEEP_KCP is set, will not stop kcp once the script is finished."
51+
else
52+
append_trap stop_kcp EXIT
53+
fi
54+
55+
# make the token available to the Go tests
56+
export KCP_AGENT_TOKEN="$(grep e2e "$KCP_TOKENFILE" | cut -f1 -d,)"
57+
58+
# Wait for kcp to be ready; this env name is also hardcoded in the Go tests.
59+
export KCP_KUBECONFIG="$KCP_ROOT_DIRECTORY/admin.kubeconfig"
60+
61+
# the tenancy API becomes available pretty late during startup, so it's a good readiness check
62+
KUBECTL="$(UGET_PRINT_PATH=relative make --no-print-directory install-kubectl)"
63+
if ! retry_linear 3 20 "$KUBECTL" --kubeconfig "$KCP_KUBECONFIG" get workspaces; then
64+
echodate "kcp never became ready."
65+
exit 1
66+
fi
67+
68+
# makes it easier to reference these files from various _test.go files.
69+
export ROOT_DIRECTORY="$(realpath .)"
70+
export KCP_KUBECONFIG="$(realpath "$KCP_KUBECONFIG")"
71+
export AGENT_BINARY="$(realpath _build/init-agent)"
72+
73+
# The tests require ARTIFACTS to be absolute.
74+
ARTIFACTS="$(realpath "$ARTIFACTS")"
75+
76+
# time to run the tests
77+
echodate "Running e2e tests…"
78+
79+
WHAT="${WHAT:-./test/e2e/...}"
80+
TEST_ARGS="${TEST_ARGS:--timeout 30m -v}"
81+
E2E_PARALLELISM=${E2E_PARALLELISM:-2}
82+
83+
(set -x; go test -tags e2e -parallel $E2E_PARALLELISM $TEST_ARGS "$WHAT")

‎internal/controller/initcontroller/controller.go‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -41,12 +41,12 @@ const (
4141
type InitTargetProvider func(ctx context.Context) (*initializationv1alpha1.InitTarget, error)
4242

4343
type Reconciler struct {
44-
remoteManager mcmanager.Manager
45-
targetProvider InitTargetProvider
46-
log *zap.SugaredLogger
47-
sourceFactory *source.Factory
48-
clusterApplier manifest.ClusterApplier
49-
initializer kcpcorev1alpha1.LogicalClusterInitializer
44+
remoteManager mcmanager.Manager
45+
targetProvider InitTargetProvider
46+
log *zap.SugaredLogger
47+
sourceFactory *source.Factory
48+
manifestApplier manifest.Applier
49+
initializer kcpcorev1alpha1.LogicalClusterInitializer
5050
}
5151

5252
// Create creates a new controller and importantly does *not* add it to the manager,
@@ -55,7 +55,7 @@ func Create(
5555
remoteManager mcmanager.Manager,
5656
targetProvider InitTargetProvider,
5757
sourceFactory *source.Factory,
58-
clusterApplier manifest.ClusterApplier,
58+
manifestApplier manifest.Applier,
5959
initializer kcpcorev1alpha1.LogicalClusterInitializer,
6060
log *zap.SugaredLogger,
6161
numWorkers int,
@@ -70,11 +70,11 @@ func Create(
7070
}).
7171
For(&kcpcorev1alpha1.LogicalCluster{}).
7272
Complete(&Reconciler{
73-
remoteManager: remoteManager,
74-
targetProvider: targetProvider,
75-
log: log.Named(ControllerName),
76-
sourceFactory: sourceFactory,
77-
clusterApplier: clusterApplier,
78-
initializer: initializer,
73+
remoteManager: remoteManager,
74+
targetProvider: targetProvider,
75+
log: log.Named(ControllerName),
76+
sourceFactory: sourceFactory,
77+
manifestApplier: manifestApplier,
78+
initializer: initializer,
7979
})
8080
}

‎internal/controller/initcontroller/reconciler.go‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -96,11 +96,6 @@ func (r *Reconciler) reconcile(ctx context.Context, logger *zap.SugaredLogger, c
9696
return requeue, fmt.Errorf("failed to get InitTarget: %w", err)
9797
}
9898

99-
applier, err := r.clusterApplier.Cluster(initialize.ClusterFromContext(ctx))
100-
if err != nil {
101-
return requeue, fmt.Errorf("failed to construct manifests applier: %w", err)
102-
}
103-
10499
for idx, ref := range target.Spec.Sources {
105100
sourceLog := logger.With("init-target", target.Name, "source-idx", idx)
106101
sourceCtx := log.WithLog(ctx, sourceLog)
@@ -117,7 +112,7 @@ func (r *Reconciler) reconcile(ctx context.Context, logger *zap.SugaredLogger, c
117112

118113
sourceLog.Debugf("Source yielded %d manifests", len(objects))
119114

120-
srcNeedRequeue, err := applier.Apply(sourceCtx, objects)
115+
srcNeedRequeue, err := r.manifestApplier.Apply(sourceCtx, client, objects)
121116
if err != nil {
122117
return requeue, fmt.Errorf("failed to apply source #%d: %w", idx, err)
123118
}

‎internal/manifest/applier.go‎

Lines changed: 8 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -21,57 +21,29 @@ import (
2121
"errors"
2222
"strings"
2323

24-
"github.com/kcp-dev/init-agent/internal/kcp"
2524
"github.com/kcp-dev/init-agent/internal/log"
2625

27-
"github.com/kcp-dev/logicalcluster/v3"
28-
2926
apierrors "k8s.io/apimachinery/pkg/api/errors"
3027
"k8s.io/apimachinery/pkg/api/meta"
3128
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
3229
ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client"
3330
)
3431

35-
type ClusterApplier interface {
36-
Cluster(cluster logicalcluster.Name) (Applier, error)
37-
}
38-
3932
type Applier interface {
40-
Apply(ctx context.Context, objs []*unstructured.Unstructured) (requeue bool, err error)
33+
Apply(ctx context.Context, client ctrlruntimeclient.Client, objs []*unstructured.Unstructured) (requeue bool, err error)
4134
}
4235

43-
type clusterApplier struct {
44-
cc kcp.ClusterClient
45-
}
36+
type applier struct{}
4637

47-
func NewClusterApplier(cc kcp.ClusterClient) ClusterApplier {
48-
return &clusterApplier{cc: cc}
49-
}
50-
51-
func (a *clusterApplier) Cluster(cluster logicalcluster.Name) (Applier, error) {
52-
client, err := a.cc.Cluster(cluster, nil) // using unstructured, no scheme needed
53-
if err != nil {
54-
return nil, err
55-
}
56-
57-
return &applier{client: client}, nil
58-
}
59-
60-
type applier struct {
61-
client ctrlruntimeclient.Client
62-
}
63-
64-
func NewApplier(client ctrlruntimeclient.Client) Applier {
65-
return &applier{
66-
client: client,
67-
}
38+
func NewApplier() Applier {
39+
return &applier{}
6840
}
6941

70-
func (a *applier) Apply(ctx context.Context, objs []*unstructured.Unstructured) (requeue bool, err error) {
42+
func (a *applier) Apply(ctx context.Context, client ctrlruntimeclient.Client, objs []*unstructured.Unstructured) (requeue bool, err error) {
7143
SortObjectsByHierarchy(objs)
7244

7345
for _, object := range objs {
74-
err := a.applyObject(ctx, object)
46+
err := a.applyObject(ctx, client, object)
7547
if err != nil {
7648
if errors.Is(err, &meta.NoKindMatchError{}) {
7749
return true, nil
@@ -84,7 +56,7 @@ func (a *applier) Apply(ctx context.Context, objs []*unstructured.Unstructured)
8456
return false, nil
8557
}
8658

87-
func (a *applier) applyObject(ctx context.Context, obj *unstructured.Unstructured) error {
59+
func (a *applier) applyObject(ctx context.Context, client ctrlruntimeclient.Client, obj *unstructured.Unstructured) error {
8860
gvk := obj.GroupVersionKind()
8961

9062
key := ctrlruntimeclient.ObjectKeyFromObject(obj).String()
@@ -94,7 +66,7 @@ func (a *applier) applyObject(ctx context.Context, obj *unstructured.Unstructure
9466
logger := log.FromContext(ctx)
9567
logger.Debugw("Applying object", "obj-key", key, "obj-gvk", gvk)
9668

97-
if err := a.client.Create(ctx, obj); err != nil {
69+
if err := client.Create(ctx, obj); err != nil {
9870
if !apierrors.IsAlreadyExists(err) {
9971
return err
10072
}

0 commit comments

Comments
 (0)