Skip to content

Commit a0c4129

Browse files
chlundechlunde
committed
Add safe-start capability
This allows cluster operators to disable namespaced or clusterwide resources, and disable database types that they do not require. * Providers create ManagedResourceDefinitions but CRDs only when activated * Users activate only needed resources through ManagedResourceActivationPolicies * Significant reduction in cluster resource overhead https://docs.crossplane.io/latest/guides/implementing-safe-start/ Signed-off-by: Carl Henrik Lunde <chlunde@ifi.uio.no>
1 parent 5702095 commit a0c4129

File tree

41 files changed

+585
-95
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+585
-95
lines changed

‎cluster/local/integration_tests.sh‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,12 @@ setup_crossplane() {
176176
setup_provider() {
177177
echo_step "installing provider"
178178

179+
echo_sub_step "applying ManagedResourceActivationPolicy to disable cluster-wide MSSQL"
180+
"${KUBECTL}" apply -f "${projectdir}/examples/activation-policy-no-cluster-mssql.yaml"
181+
182+
echo_sub_step "deleting default ManagedResourceActivationPolicy that activates everything"
183+
"${KUBECTL}" delete managedresourceactivationpolicy default --ignore-not-found=true
184+
179185
local yaml="$( cat <<EOF
180186
apiVersion: pkg.crossplane.io/v1beta1
181187
kind: DeploymentRuntimeConfig
@@ -212,13 +218,32 @@ EOF
212218

213219
echo_step "waiting for provider to be installed"
214220
"${KUBECTL}" wait "provider.pkg.crossplane.io/${PACKAGE_NAME}" --for=condition=healthy --timeout=60s
221+
222+
echo_step "verifying cluster-wide MSSQL CRDs are NOT installed"
223+
if "${KUBECTL}" get crd databases.mssql.sql.crossplane.io 2>/dev/null; then
224+
echo_error "cluster-wide MSSQL Database CRD should not be installed"
225+
fi
226+
if "${KUBECTL}" get crd grants.mssql.sql.crossplane.io 2>/dev/null; then
227+
echo_error "cluster-wide MSSQL Grant CRD should not be installed"
228+
fi
229+
if "${KUBECTL}" get crd users.mssql.sql.crossplane.io 2>/dev/null; then
230+
echo_error "cluster-wide MSSQL User CRD should not be installed"
231+
fi
232+
echo_step_completed
233+
234+
echo_step "verifying namespaced MSSQL CRDs ARE installed"
235+
"${KUBECTL}" get crd databases.mssql.sql.m.crossplane.io || echo_error "namespaced MSSQL Database CRD should be installed"
236+
"${KUBECTL}" get crd grants.mssql.sql.m.crossplane.io || echo_error "namespaced MSSQL Grant CRD should be installed"
237+
"${KUBECTL}" get crd users.mssql.sql.m.crossplane.io || echo_error "namespaced MSSQL User CRD should be installed"
238+
echo_step_completed
215239
}
216240

217241
cleanup_provider() {
218242
echo_step "uninstalling provider"
219243

220244
"${KUBECTL}" delete provider.pkg.crossplane.io "${PACKAGE_NAME}"
221245
"${KUBECTL}" delete deploymentruntimeconfig.pkg.crossplane.io debug-config
246+
"${KUBECTL}" delete managedresourceactivationpolicy.apiextensions.crossplane.io disable-cluster-mssql --ignore-not-found=true
222247

223248
echo_step "waiting for provider pods to be deleted"
224249
timeout=60

‎cmd/provider/main.go‎

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ limitations under the License.
1717
package main
1818

1919
import (
20+
"context"
2021
"os"
2122
"path/filepath"
2223

@@ -25,19 +26,46 @@ import (
2526
_ "github.com/lib/pq"
2627

2728
"github.com/alecthomas/kingpin/v2"
29+
authv1 "k8s.io/api/authorization/v1"
30+
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
31+
"k8s.io/apimachinery/pkg/runtime/schema"
2832
ctrl "sigs.k8s.io/controller-runtime"
2933
"sigs.k8s.io/controller-runtime/pkg/cache"
34+
"sigs.k8s.io/controller-runtime/pkg/client"
3035
"sigs.k8s.io/controller-runtime/pkg/log/zap"
3136

3237
xpcontroller "github.com/crossplane/crossplane-runtime/v2/pkg/controller"
3338
"github.com/crossplane/crossplane-runtime/v2/pkg/feature"
39+
"github.com/crossplane/crossplane-runtime/v2/pkg/gate"
3440
"github.com/crossplane/crossplane-runtime/v2/pkg/logging"
3541
"github.com/crossplane/crossplane-runtime/v2/pkg/ratelimiter"
42+
"github.com/crossplane/crossplane-runtime/v2/pkg/reconciler/customresourcesgate"
3643

3744
"github.com/crossplane-contrib/provider-sql/apis"
3845
"github.com/crossplane-contrib/provider-sql/pkg/controller"
3946
)
4047

48+
// canWatchCRD checks if the provider has the necessary RBAC permissions to watch CustomResourceDefinitions.
49+
// This is used to determine if we can use the SafeStart pattern with gated controller initialization.
50+
func canWatchCRD(ctx context.Context, c client.Client) (bool, error) {
51+
review := &authv1.SelfSubjectAccessReview{
52+
Spec: authv1.SelfSubjectAccessReviewSpec{
53+
ResourceAttributes: &authv1.ResourceAttributes{
54+
Group: apiextensionsv1.GroupName,
55+
Version: "*",
56+
Resource: "customresourcedefinitions",
57+
Verb: "list",
58+
},
59+
},
60+
}
61+
62+
if err := c.Create(ctx, review); err != nil {
63+
return false, err
64+
}
65+
66+
return review.Status.Allowed, nil
67+
}
68+
4169
func main() {
4270
var (
4371
app = kingpin.New(filepath.Base(os.Args[0]), "SQL support for Crossplane.").DefaultEnvars()
@@ -73,6 +101,7 @@ func main() {
73101
})
74102
kingpin.FatalIfError(err, "Cannot create controller manager")
75103
kingpin.FatalIfError(apis.AddToScheme(mgr.GetScheme()), "Cannot add SQL APIs to scheme")
104+
kingpin.FatalIfError(apiextensionsv1.AddToScheme(mgr.GetScheme()), "Cannot add CRD types to scheme")
76105

77106
o := xpcontroller.Options{
78107
Logger: log,
@@ -86,6 +115,23 @@ func main() {
86115
log.Info("Beta feature enabled", "flag", feature.EnableBetaManagementPolicies)
87116
}
88117

89-
kingpin.FatalIfError(controller.Setup(mgr, o), "Cannot setup SQL controllers")
118+
// Check if we have permission to watch CRDs for SafeStart support
119+
ctx := context.Background()
120+
canWatch, err := canWatchCRD(ctx, mgr.GetClient())
121+
switch {
122+
case err != nil:
123+
log.Info("Failed to check CRD watch permissions, using immediate controller setup", "error", err)
124+
kingpin.FatalIfError(controller.Setup(mgr, o), "Cannot setup SQL controllers")
125+
case canWatch:
126+
log.Info("SafeStart enabled: using gated controller initialization")
127+
o.Gate = new(gate.Gate[schema.GroupVersionKind])
128+
129+
kingpin.FatalIfError(customresourcesgate.Setup(mgr, o), "Cannot setup CRD gate")
130+
kingpin.FatalIfError(controller.SetupGated(mgr, o), "Cannot setup SQL controllers")
131+
default:
132+
log.Info("SafeStart disabled: insufficient CRD watch permissions, using immediate controller setup")
133+
kingpin.FatalIfError(controller.Setup(mgr, o), "Cannot setup SQL controllers")
134+
}
135+
90136
kingpin.FatalIfError(mgr.Start(ctrl.SetupSignalHandler()), "Cannot start controller manager")
91137
}

‎examples/activation-policy-no-cluster-mssql.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
apiVersion: apiextensions.crossplane.io/v1alpha1
2+
kind: ManagedResourceActivationPolicy
3+
metadata:
4+
name: disable-cluster-mssql
5+
spec:
6+
activate:
7+
# Activate all MySQL resources (both cluster and namespaced)
8+
- "*.mysql.sql.crossplane.io"
9+
- "*.mysql.sql.m.crossplane.io"
10+
# Activate all PostgreSQL resources (both cluster and namespaced)
11+
- "*.postgresql.sql.crossplane.io"
12+
- "*.postgresql.sql.m.crossplane.io"
13+
# Activate only namespaced MSSQL resources (cluster-wide MSSQL is excluded)
14+
- "*.mssql.sql.m.crossplane.io"

‎go.mod‎

Lines changed: 25 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -5,18 +5,19 @@ go 1.24.7
55
require (
66
github.com/DATA-DOG/go-sqlmock v1.5.2
77
github.com/alecthomas/kingpin/v2 v2.4.0
8-
github.com/crossplane/crossplane-runtime/v2 v2.0.0
8+
github.com/crossplane/crossplane-runtime/v2 v2.1.0
99
github.com/crossplane/crossplane-tools v0.0.0-20250731192036-00d407d8b7ec
1010
github.com/crossplane/upjet/v2 v2.1.0
1111
github.com/denisenkom/go-mssqldb v0.11.0
1212
github.com/go-sql-driver/mysql v1.9.3
1313
github.com/google/go-cmp v0.7.0
1414
github.com/lib/pq v1.10.9
1515
github.com/pkg/errors v0.9.1
16-
k8s.io/api v0.33.0
17-
k8s.io/apimachinery v0.33.0
18-
k8s.io/utils v0.0.0-20250321185631-1f6e0b77f77e
19-
sigs.k8s.io/controller-runtime v0.19.0
16+
k8s.io/api v0.34.0
17+
k8s.io/apiextensions-apiserver v0.34.0
18+
k8s.io/apimachinery v0.34.0
19+
k8s.io/utils v0.0.0-20250604170112-4c0f3b243397
20+
sigs.k8s.io/controller-runtime v0.22.0
2021
sigs.k8s.io/controller-tools v0.18.0
2122
)
2223

@@ -33,13 +34,13 @@ require (
3334
github.com/cespare/xxhash/v2 v2.3.0 // indirect
3435
github.com/dave/jennifer v1.7.1 // indirect
3536
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
36-
github.com/emicklei/go-restful/v3 v3.12.1 // indirect
37+
github.com/emicklei/go-restful/v3 v3.12.2 // indirect
3738
github.com/evanphx/json-patch v5.9.11+incompatible // indirect
38-
github.com/evanphx/json-patch/v5 v5.9.0 // indirect
39+
github.com/evanphx/json-patch/v5 v5.9.11 // indirect
3940
github.com/fatih/color v1.18.0 // indirect
40-
github.com/fsnotify/fsnotify v1.8.0 // indirect
41-
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
42-
github.com/go-logr/logr v1.4.2 // indirect
41+
github.com/fsnotify/fsnotify v1.9.0 // indirect
42+
github.com/fxamacker/cbor/v2 v2.9.0 // indirect
43+
github.com/go-logr/logr v1.4.3 // indirect
4344
github.com/go-logr/zapr v1.3.0 // indirect
4445
github.com/go-openapi/jsonpointer v0.21.0 // indirect
4546
github.com/go-openapi/jsonreference v0.21.0 // indirect
@@ -48,7 +49,8 @@ require (
4849
github.com/gogo/protobuf v1.3.2 // indirect
4950
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
5051
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
51-
github.com/google/gnostic-models v0.6.9 // indirect
52+
github.com/google/btree v1.1.3 // indirect
53+
github.com/google/gnostic-models v0.7.0 // indirect
5254
github.com/google/uuid v1.6.0 // indirect
5355
github.com/hashicorp/hcl/v2 v2.23.0 // indirect
5456
github.com/inconshreveable/mousetrap v1.1.0 // indirect
@@ -59,8 +61,9 @@ require (
5961
github.com/mattn/go-isatty v0.0.20 // indirect
6062
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
6163
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
62-
github.com/modern-go/reflect2 v1.0.2 // indirect
64+
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
6365
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
66+
github.com/pmezard/go-difflib v1.0.0 // indirect
6467
github.com/prometheus/client_golang v1.22.0 // indirect
6568
github.com/prometheus/client_model v0.6.1 // indirect
6669
github.com/prometheus/common v0.62.0 // indirect
@@ -77,8 +80,9 @@ require (
7780
go.opentelemetry.io/otel/trace v1.35.0 // indirect
7881
go.uber.org/multierr v1.11.0 // indirect
7982
go.uber.org/zap v1.27.0 // indirect
83+
go.yaml.in/yaml/v2 v2.4.2 // indirect
84+
go.yaml.in/yaml/v3 v3.0.4 // indirect
8085
golang.org/x/crypto v0.45.0 // indirect
81-
golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect
8286
golang.org/x/mod v0.29.0 // indirect
8387
golang.org/x/net v0.47.0 // indirect
8488
golang.org/x/oauth2 v0.29.0 // indirect
@@ -90,22 +94,21 @@ require (
9094
golang.org/x/tools v0.38.0 // indirect
9195
golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated // indirect
9296
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
93-
google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a // indirect
97+
google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb // indirect
9498
google.golang.org/grpc v1.72.1 // indirect
9599
google.golang.org/protobuf v1.36.6 // indirect
96100
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
97101
gopkg.in/inf.v0 v0.9.1 // indirect
98102
gopkg.in/yaml.v2 v2.4.0 // indirect
99103
gopkg.in/yaml.v3 v3.0.1 // indirect
100-
k8s.io/apiextensions-apiserver v0.33.0 // indirect
101-
k8s.io/client-go v0.33.0 // indirect
102-
k8s.io/code-generator v0.33.0 // indirect
103-
k8s.io/component-base v0.33.0 // indirect
104-
k8s.io/gengo/v2 v2.0.0-20250207200755-1244d31929d7 // indirect
104+
k8s.io/client-go v0.34.0 // indirect
105+
k8s.io/code-generator v0.34.0 // indirect
106+
k8s.io/component-base v0.34.0 // indirect
107+
k8s.io/gengo/v2 v2.0.0-20250604051438-85fd79dbfd9f // indirect
105108
k8s.io/klog/v2 v2.130.1 // indirect
106-
k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect
109+
k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect
107110
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
108111
sigs.k8s.io/randfill v1.0.0 // indirect
109-
sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect
110-
sigs.k8s.io/yaml v1.4.0 // indirect
112+
sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect
113+
sigs.k8s.io/yaml v1.6.0 // indirect
111114
)

0 commit comments

Comments
 (0)