Skip to content

Commit 215b37e

Browse files
ecordellecordell
committed
report pod errors on the spicedbcluster status
1 parent 76d34cc commit 215b37e

File tree

10 files changed

+311
-134
lines changed

10 files changed

+311
-134
lines changed

‎e2e/cluster_test.go‎

Lines changed: 70 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ var _ = Describe("SpiceDBClusters", func() {
5151
mapper meta.RESTMapper
5252
testNamespace string
5353
image string
54+
config map[string]any
5455
secret *corev1.Secret
5556
cluster *v1alpha1.SpiceDBCluster
5657

@@ -100,15 +101,14 @@ var _ = Describe("SpiceDBClusters", func() {
100101
},
101102
}
102103

103-
config := map[string]any{
104+
config = map[string]any{
104105
"envPrefix": spicedbEnvPrefix,
105106
"image": image,
106107
"cmd": spicedbCmd,
107108
"skipReleaseCheck": "true",
108109
"telemetryEndpoint": "",
109110
}
110-
jsonConfig, err := json.Marshal(config)
111-
Expect(err).To(Succeed())
111+
112112
cluster = &v1alpha1.SpiceDBCluster{
113113
TypeMeta: metav1.TypeMeta{
114114
Kind: v1alpha1.SpiceDBClusterKind,
@@ -119,7 +119,6 @@ var _ = Describe("SpiceDBClusters", func() {
119119
Namespace: testNamespace,
120120
},
121121
Spec: v1alpha1.ClusterSpec{
122-
Config: jsonConfig,
123122
SecretRef: "spicedb",
124123
},
125124
}
@@ -136,6 +135,10 @@ var _ = Describe("SpiceDBClusters", func() {
136135
// be modified by nested BeforeEach blocks.
137136
_, err := kclient.CoreV1().Secrets(testNamespace).Create(ctx, secret, metav1.CreateOptions{})
138137
Expect(err).To(Succeed())
138+
139+
jsonConfig, err := json.Marshal(config)
140+
Expect(err).To(Succeed())
141+
cluster.Spec.Config = jsonConfig
139142
u, err := runtime.DefaultUnstructuredConverter.ToUnstructured(cluster)
140143
Expect(err).To(Succeed())
141144
_, err = client.Resource(v1alpha1ClusterGVR).Namespace(cluster.Namespace).Create(ctx, &unstructured.Unstructured{Object: u}, metav1.CreateOptions{})
@@ -233,7 +236,7 @@ var _ = Describe("SpiceDBClusters", func() {
233236
"migration_secrets": "kaitain-bootstrap-token=testtesttesttest,sharewith-bootstrap-token=testtesttesttest,thumper-bootstrap-token=testtesttesttest,metrics-proxy-token=testtesttesttest",
234237
}
235238

236-
config := map[string]any{
239+
config = map[string]any{
237240
"skipReleaseCheck": "true",
238241
"telemetryEndpoint": "",
239242
"datastoreEngine": db.Engine,
@@ -246,10 +249,6 @@ var _ = Describe("SpiceDBClusters", func() {
246249
for k, v := range db.ExtraConfig {
247250
config[k] = v
248251
}
249-
jsonConfig, err := json.Marshal(config)
250-
Expect(err).To(Succeed())
251-
252-
cluster.Spec.Config = jsonConfig
253252
})
254253

255254
JustBeforeEach(func() {
@@ -271,7 +270,7 @@ var _ = Describe("SpiceDBClusters", func() {
271270
BeforeEach(func() {
272271
// this installs from the head of the current channel, skip validating image
273272
image = ""
274-
config := map[string]any{
273+
config = map[string]any{
275274
"skipReleaseCheck": true,
276275
"telemetryEndpoint": "",
277276
"datastoreEngine": db.Engine,
@@ -287,9 +286,6 @@ var _ = Describe("SpiceDBClusters", func() {
287286
for k, v := range db.ExtraConfig {
288287
config[k] = v
289288
}
290-
jsonConfig, err := json.Marshal(config)
291-
Expect(err).To(BeNil())
292-
cluster.Spec.Config = jsonConfig
293289
cluster.Spec.Patches = []v1alpha1.Patch{{
294290
Kind: "Deployment",
295291
Patch: json.RawMessage(`{
@@ -305,7 +301,7 @@ var _ = Describe("SpiceDBClusters", func() {
305301
ktypes.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace},
306302
ktypes.NamespacedName{Name: "spicedb-grpc-tls", Namespace: cluster.Namespace},
307303
)
308-
_, err = kclient.CoreV1().Secrets(cluster.Namespace).Create(ctx, tlsSecret, metav1.CreateOptions{})
304+
_, err := kclient.CoreV1().Secrets(cluster.Namespace).Create(ctx, tlsSecret, metav1.CreateOptions{})
309305
Expect(err).To(Succeed())
310306
DeferCleanup(kclient.CoreV1().Secrets(cluster.Namespace).Delete, ctx, tlsSecret.Name, metav1.DeleteOptions{})
311307
})
@@ -329,6 +325,59 @@ var _ = Describe("SpiceDBClusters", func() {
329325
})
330326
})
331327

328+
When("a valid spicedb cluster with a pod error", func() {
329+
BeforeEach(func() {
330+
secret.StringData = map[string]string{
331+
"datastore_uri": db.DatastoreURI,
332+
"preshared_key": "testtesttesttest",
333+
"migration_secrets": "kaitain-bootstrap-token=testtesttesttest,sharewith-bootstrap-token=testtesttesttest,thumper-bootstrap-token=testtesttesttest,metrics-proxy-token=testtesttesttest",
334+
}
335+
config = map[string]any{
336+
"skipMigrations": true,
337+
"datastoreEngine": db.Engine,
338+
"image": image,
339+
"envPrefix": spicedbEnvPrefix,
340+
"cmd": "badcmd",
341+
"datastoreConnpoolReadMinOpen": "1",
342+
"datastoreConnpoolWriteMinOpen": "1",
343+
}
344+
for k, v := range db.ExtraConfig {
345+
config[k] = v
346+
}
347+
})
348+
349+
It("reports un-recovered pod errors on the status", func() {
350+
var lastCluster *v1alpha1.SpiceDBCluster
351+
var condition *metav1.Condition
352+
Watch(ctx, client, v1alpha1ClusterGVR, ktypes.NamespacedName{Name: cluster.Name, Namespace: testNamespace}, "0", func(c *v1alpha1.SpiceDBCluster) bool {
353+
condition = c.FindStatusCondition(v1alpha1.ConditionTypeRolloutError)
354+
logr.FromContextOrDiscard(ctx).Info("watch event", "status", c.Status)
355+
lastCluster = c
356+
return condition == nil
357+
})
358+
Expect(condition).To(EqualCondition(v1alpha1.NewPodErrorCondition(`failed to create containerd task: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: exec: "badcmd": executable file not found in $PATH: unknown`)))
359+
360+
By("fixing the config problem")
361+
config["cmd"] = spicedbCmd
362+
config["skipMigrations"] = "false"
363+
jsonConfig, err := json.Marshal(config)
364+
Expect(err).To(Succeed())
365+
lastCluster.Spec.Config = jsonConfig
366+
u, err := runtime.DefaultUnstructuredConverter.ToUnstructured(lastCluster)
367+
Expect(err).To(Succeed())
368+
_, err = client.Resource(v1alpha1ClusterGVR).Namespace(cluster.Namespace).Update(ctx, &unstructured.Unstructured{Object: u}, metav1.UpdateOptions{})
369+
Expect(err).To(Succeed())
370+
371+
By("the condition should be removed")
372+
Watch(ctx, client, v1alpha1ClusterGVR, ktypes.NamespacedName{Name: cluster.Name, Namespace: testNamespace}, "0", func(c *v1alpha1.SpiceDBCluster) bool {
373+
condition = c.FindStatusCondition(v1alpha1.ConditionTypeRolloutError)
374+
logr.FromContextOrDiscard(ctx).Info("watch event", "status", c.Status)
375+
return condition != nil
376+
})
377+
Expect(condition).To(BeNil())
378+
})
379+
})
380+
332381
When("a valid SpiceDBCluster and skipped migrations", func() {
333382
BeforeEach(func() {
334383
secret.StringData = map[string]string{
@@ -337,17 +386,15 @@ var _ = Describe("SpiceDBClusters", func() {
337386
"preshared_key": "testtesttesttest",
338387
"migration_secrets": "kaitain-bootstrap-token=testtesttesttest,sharewith-bootstrap-token=testtesttesttest,thumper-bootstrap-token=testtesttesttest,metrics-proxy-token=testtesttesttest",
339388
}
340-
config, err := json.Marshal(map[string]any{
389+
config = map[string]any{
341390
"skipMigrations": true,
342391
"datastoreEngine": db.Engine,
343392
"image": image,
344393
"envPrefix": spicedbEnvPrefix,
345394
"cmd": spicedbCmd,
346-
"datastoreConnpoolReadMinOpen": 1,
347-
"datastoreConnpoolWriteMinOpen": 1,
348-
})
349-
Expect(err).To(Succeed())
350-
cluster.Spec.Config = config
395+
"datastoreConnpoolReadMinOpen": "1",
396+
"datastoreConnpoolWriteMinOpen": "1",
397+
}
351398
})
352399

353400
It("Starts SpiceDB without migrating", func() {
@@ -381,7 +428,7 @@ var _ = Describe("SpiceDBClusters", func() {
381428
"migration_secrets": "kaitain-bootstrap-token=testtesttesttest,sharewith-bootstrap-token=testtesttesttest,thumper-bootstrap-token=testtesttesttest,metrics-proxy-token=testtesttesttest",
382429
}
383430

384-
config := map[string]any{
431+
config = map[string]any{
385432
"skipReleaseCheck": "true",
386433
"telemetryEndpoint": "",
387434
"datastoreEngine": engine,
@@ -400,10 +447,6 @@ var _ = Describe("SpiceDBClusters", func() {
400447
}
401448
config[k] = v
402449
}
403-
jsonConfig, err := json.Marshal(config)
404-
Expect(err).To(Succeed())
405-
406-
cluster.Spec.Config = jsonConfig
407450
cluster.Spec.Channel = channel
408451
cluster.Spec.Version = from
409452
})
@@ -527,18 +570,15 @@ var _ = Describe("SpiceDBClusters", func() {
527570

528571
Describe("there is a series of required migrations", Label("published"), func() {
529572
BeforeEach(func() {
530-
classConfig := map[string]any{
573+
config = map[string]any{
531574
"skipReleaseCheck": "true",
532575
"telemetryEndpoint": "",
533576
"logLevel": "debug",
534577
"datastoreEngine": "postgres",
535578
"tlsSecretName": "spicedb-grpc-tls",
536579
"dispatchUpstreamCASecretName": "spicedb-grpc-tls",
537580
}
538-
jsonConfig, err := json.Marshal(classConfig)
539-
Expect(err).To(BeNil())
540581
cluster.Spec.Version = "v1.13.0"
541-
cluster.Spec.Config = jsonConfig
542582

543583
secret.StringData = map[string]string{
544584
"datastore_uri": db.DatastoreURI,
@@ -550,7 +590,7 @@ var _ = Describe("SpiceDBClusters", func() {
550590
ktypes.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace},
551591
ktypes.NamespacedName{Name: "spicedb-grpc-tls", Namespace: cluster.Namespace},
552592
)
553-
_, err = kclient.CoreV1().Secrets(cluster.Namespace).Create(ctx, tlsSecret, metav1.CreateOptions{})
593+
_, err := kclient.CoreV1().Secrets(cluster.Namespace).Create(ctx, tlsSecret, metav1.CreateOptions{})
554594
Expect(err).To(Succeed())
555595
DeferCleanup(kclient.CoreV1().Secrets(cluster.Namespace).Delete, ctx, tlsSecret.Name, metav1.DeleteOptions{})
556596
})

‎e2e/e2e_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -190,7 +190,7 @@ func StartOperator(rc *rest.Config) {
190190
go func() {
191191
defer GinkgoRecover()
192192
options := run.RecommendedOptions()
193-
options.DebugAddress = ":"
193+
options.DebugAddress = "localhost:"
194194
options.BootstrapCRDs = true
195195
options.OperatorConfigPath = WriteConfig(GetConfig(ProposedGraphFile))
196196
_ = options.Run(ctx, cmdutil.NewFactory(ClientGetter{config: rc}))

‎go.mod‎

Lines changed: 24 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ require (
66
cloud.google.com/go/spanner v1.39.0
77
github.com/authzed/controller-idioms v0.7.0
88
github.com/cespare/xxhash/v2 v2.1.2
9-
github.com/evanphx/json-patch v4.12.0+incompatible
9+
github.com/evanphx/json-patch v5.6.0+incompatible
1010
github.com/fatih/camelcase v1.0.0
1111
github.com/fluxcd/pkg/ssa v0.23.0
1212
github.com/go-logr/logr v1.2.3
@@ -29,17 +29,17 @@ require (
2929
google.golang.org/genproto v0.0.0-20220916134934-764224ccc2d1
3030
google.golang.org/grpc v1.49.0
3131
k8s.io/api v0.26.3
32-
k8s.io/apiextensions-apiserver v0.26.0
32+
k8s.io/apiextensions-apiserver v0.26.2
3333
k8s.io/apimachinery v0.26.3
34-
k8s.io/apiserver v0.26.0
34+
k8s.io/apiserver v0.26.2
3535
k8s.io/cli-runtime v0.26.3
3636
k8s.io/client-go v0.26.3
37-
k8s.io/code-generator v0.26.0
37+
k8s.io/code-generator v0.26.2
3838
k8s.io/component-base v0.26.3
3939
k8s.io/controller-manager v0.26.0
40-
k8s.io/klog/v2 v2.80.1
40+
k8s.io/klog/v2 v2.90.1
4141
k8s.io/kubectl v0.26.3
42-
k8s.io/utils v0.0.0-20221128185143-99ec85e7a448
42+
k8s.io/utils v0.0.0-20230313181309-38a27ef9d749
4343
mvdan.cc/gofumpt v0.4.0
4444
sigs.k8s.io/cli-utils v0.34.0
4545
sigs.k8s.io/controller-runtime v0.14.1
@@ -54,7 +54,7 @@ require (
5454
cloud.google.com/go v0.102.1 // indirect
5555
cloud.google.com/go/compute v1.7.0 // indirect
5656
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
57-
github.com/BurntSushi/toml v1.0.0 // indirect
57+
github.com/BurntSushi/toml v1.2.1 // indirect
5858
github.com/MakeNowJust/heredoc v1.0.0 // indirect
5959
github.com/NYTimes/gziphandler v1.1.1 // indirect
6060
github.com/alessio/shellescape v1.4.1 // indirect
@@ -70,7 +70,7 @@ require (
7070
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
7171
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
7272
github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
73-
github.com/fatih/color v1.12.0 // indirect
73+
github.com/fatih/color v1.13.0 // indirect
7474
github.com/felixge/httpsnoop v1.0.3 // indirect
7575
github.com/fsnotify/fsnotify v1.6.0 // indirect
7676
github.com/fvbommel/sortorder v1.0.1 // indirect
@@ -85,7 +85,7 @@ require (
8585
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
8686
github.com/golang/protobuf v1.5.2 // indirect
8787
github.com/google/btree v1.0.1 // indirect
88-
github.com/google/cel-go v0.12.5 // indirect
88+
github.com/google/cel-go v0.12.6 // indirect
8989
github.com/google/gnostic v0.5.7-v3refs // indirect
9090
github.com/google/go-cmp v0.5.9 // indirect
9191
github.com/google/gofuzz v1.2.0 // indirect
@@ -98,20 +98,20 @@ require (
9898
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
9999
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
100100
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
101-
github.com/imdario/mergo v0.3.12 // indirect
101+
github.com/imdario/mergo v0.3.14 // indirect
102102
github.com/inconshreveable/mousetrap v1.0.1 // indirect
103103
github.com/jackc/pgpassfile v1.0.0 // indirect
104104
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
105105
github.com/josharian/intern v1.0.0 // indirect
106106
github.com/json-iterator/go v1.1.12 // indirect
107107
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
108108
github.com/mailru/easyjson v0.7.6 // indirect
109-
github.com/mattn/go-colorable v0.1.8 // indirect
110-
github.com/mattn/go-isatty v0.0.14 // indirect
111-
github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect
109+
github.com/mattn/go-colorable v0.1.13 // indirect
110+
github.com/mattn/go-isatty v0.0.17 // indirect
111+
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
112112
github.com/mitchellh/go-wordwrap v1.0.0 // indirect
113113
github.com/moby/spdystream v0.2.0 // indirect
114-
github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect
114+
github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect
115115
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
116116
github.com/modern-go/reflect2 v1.0.2 // indirect
117117
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
@@ -125,6 +125,7 @@ require (
125125
github.com/prometheus/common v0.37.0 // indirect
126126
github.com/prometheus/procfs v0.8.0 // indirect
127127
github.com/russross/blackfriday/v2 v2.1.0 // indirect
128+
github.com/sirupsen/logrus v1.9.0 // indirect
128129
github.com/spf13/pflag v1.0.5 // indirect
129130
github.com/stoewer/go-strcase v1.2.0 // indirect
130131
github.com/xlab/treeprint v1.1.0 // indirect
@@ -144,25 +145,25 @@ require (
144145
go.opentelemetry.io/proto/otlp v0.19.0 // indirect
145146
go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
146147
go.uber.org/multierr v1.6.0 // indirect
147-
golang.org/x/crypto v0.1.0 // indirect
148-
golang.org/x/mod v0.7.0 // indirect
149-
golang.org/x/net v0.7.0 // indirect
148+
golang.org/x/crypto v0.5.0 // indirect
149+
golang.org/x/mod v0.8.0 // indirect
150+
golang.org/x/net v0.8.0 // indirect
150151
golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 // indirect
151152
golang.org/x/sync v0.1.0 // indirect
152-
golang.org/x/sys v0.5.0 // indirect
153-
golang.org/x/term v0.5.0 // indirect
154-
golang.org/x/text v0.7.0 // indirect
153+
golang.org/x/sys v0.6.0 // indirect
154+
golang.org/x/term v0.6.0 // indirect
155+
golang.org/x/text v0.8.0 // indirect
155156
golang.org/x/time v0.3.0 // indirect
156-
golang.org/x/tools v0.5.0 // indirect
157+
golang.org/x/tools v0.6.0 // indirect
157158
google.golang.org/appengine v1.6.7 // indirect
158159
google.golang.org/protobuf v1.28.1 // indirect
159160
gopkg.in/inf.v0 v0.9.1 // indirect
160161
gopkg.in/yaml.v2 v2.4.0 // indirect
161162
gopkg.in/yaml.v3 v3.0.1 // indirect
162163
k8s.io/gengo v0.0.0-20220902162205-c0856e24416d // indirect
163164
k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
164-
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.33 // indirect
165-
sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
165+
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35 // indirect
166+
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
166167
sigs.k8s.io/kustomize/api v0.12.1 // indirect
167168
sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect
168169
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect

0 commit comments

Comments
 (0)