Skip to content

Commit 62cc41a

Browse files
johscheuerjohscheuer
authored
Document how to run e2e tests with the unified image (#1998)
* Document how to run e2e tests with the unified image
1 parent 8575350 commit 62cc41a

File tree

4 files changed

+166
-10
lines changed

4 files changed

+166
-10
lines changed

‎e2e/README.md

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,26 @@ FDB_VERSION=7.1.33 make -C e2e -kj test_operator.run
5656

5757
If those tests are running on a cluster that has no chaos-mesh installed, you can set `ENABLE_CHAOS_TESTS=false` to disable all test that uses chaos-mesh.
5858

59+
### Running the e2e tests with the unified image
60+
61+
_NOTE_ The unified image is currently experimental and there are no pre-build images.
62+
63+
The source can be found in the [FoundationDB repository](https://github.com/apple/foundationdb/tree/main/fdbkubernetesmonitor#foundationdb-kubernetes-monitor).
64+
You have to make sure that the image `foundationdb/fdb-kubernetes-monitor` is built and available in the Kubernetes cluster.
65+
66+
```bash
67+
FDB_IMAGE=foundationdb/fdb-kubernetes-monitor:7.3.38 \
68+
SIDECAR_IMAGE=foundationdb/fdb-kubernetes-monitor:7.3.38 \
69+
FEATURE_UNIFIED_IMAGE=true \
70+
UPGRADE_VERSIONS="" \
71+
FDB_VERSION="7.3.38" \
72+
make -C e2e test_operator.run
73+
```
74+
5975
### Running e2e tests in kind
6076

77+
_NOTE_ This setup is currently not used by our CI.
78+
6179
[kind](https://kind.sigs.k8s.io) provides an easy way to run a local Kubernetes cluster.
6280
For running tests on a `kind` cluster you should set the `CLOUD_PROVIDER=kind` environment variable to make sure the test framework is creating clusters with smaller resource requirements.
6381
The following steps assume that `kind` and `helm` are already installed.

‎e2e/fixtures/fdb_data_loader.go

Lines changed: 122 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,123 @@ spec:
161161
- name: fdb-certs
162162
secret:
163163
secretName: {{ .SecretName }}`
164+
165+
// For now we only load 2GB into the cluster, we can increase this later if we want.
166+
dataLoaderJobUnifiedImage = `apiVersion: batch/v1
167+
kind: Job
168+
metadata:
169+
name: {{ .Name }}
170+
namespace: {{ .Namespace }}
171+
labels:
172+
app: {{ .Name }}
173+
spec:
174+
backoffLimit: 2
175+
completions: 2
176+
parallelism: 2
177+
template:
178+
spec:
179+
containers:
180+
- image: {{ .Image }}
181+
imagePullPolicy: Always
182+
name: {{ .Name }}
183+
# This configuration will load ~1GB per data loader.
184+
args:
185+
- --keys=1000000
186+
- --batch-size=50
187+
- --value-size=1000
188+
env:
189+
- name: FDB_CLUSTER_FILE
190+
value: /var/dynamic/fdb/fdb.cluster
191+
- name: FDB_TLS_CERTIFICATE_FILE
192+
value: /tmp/fdb-certs/tls.crt
193+
- name: FDB_TLS_CA_FILE
194+
value: /tmp/fdb-certs/ca.pem
195+
- name: FDB_TLS_KEY_FILE
196+
value: /tmp/fdb-certs/tls.key
197+
# FDB 7.3 adds a check for loading external client library, which doesn't work with 6.3.
198+
# Consider remove this option once 6.3 is no longer being used.
199+
- name: FDB_NETWORK_OPTION_IGNORE_EXTERNAL_CLIENT_FAILURES
200+
value: ""
201+
- name: LD_LIBRARY_PATH
202+
value: /var/dynamic/fdb
203+
- name: FDB_NETWORK_OPTION_TRACE_LOG_GROUP
204+
value: {{ .Name }}
205+
- name: FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY
206+
value: /var/dynamic/fdb
207+
- name: PYTHONUNBUFFERED
208+
value: "on"
209+
volumeMounts:
210+
- name: config-map
211+
mountPath: /var/dynamic-conf
212+
- name: fdb-libs
213+
mountPath: /var/dynamic/fdb
214+
- name: fdb-certs
215+
mountPath: /tmp/fdb-certs
216+
readOnly: true
217+
resources:
218+
requests:
219+
cpu: "1"
220+
memory: 4Gi
221+
initContainers:
222+
{{ range $index, $version := .SidecarVersions }}
223+
- name: foundationdb-kubernetes-init-{{ $index }}
224+
image: {{ .BaseImage }}:{{ .SidecarTag}}
225+
imagePullPolicy: {{ .ImagePullPolicy }}
226+
args:
227+
- --mode
228+
- init
229+
- --output-dir
230+
- /var/output-files
231+
- --copy-library
232+
- "{{ .FDBVersion.Compact }}"
233+
{{ if .CopyAsPrimary }}
234+
- --copy-primary-library
235+
- "{{ .FDBVersion.Compact }}"
236+
{{ end }}
237+
volumeMounts:
238+
- name: fdb-libs
239+
mountPath: /var/output-files
240+
securityContext:
241+
runAsUser: 0
242+
runAsGroup: 0
243+
{{ if .CopyAsPrimary }}
244+
- name: foundationdb-kubernetes-init-cluster-file
245+
image: {{ .BaseImage }}:{{ .SidecarTag}}
246+
imagePullPolicy: {{ .ImagePullPolicy }}
247+
args:
248+
- --mode
249+
- init
250+
- --input-dir
251+
- /var/dynamic-conf
252+
- --output-dir
253+
- /var/output-files
254+
- --copy-file
255+
- fdb.cluster
256+
- --require-not-empty
257+
- fdb.cluster
258+
volumeMounts:
259+
- name: fdb-libs
260+
mountPath: /var/output-files
261+
- name: config-map
262+
mountPath: /var/dynamic-conf
263+
securityContext:
264+
runAsUser: 0
265+
runAsGroup: 0
266+
{{ end }}
267+
{{ end }}
268+
restartPolicy: Never
269+
volumes:
270+
- name: config-map
271+
configMap:
272+
name: {{ .ClusterName }}-config
273+
items:
274+
- key: cluster-file
275+
path: fdb.cluster
276+
- name: fdb-libs
277+
emptyDir: {}
278+
- name: fdb-certs
279+
secret:
280+
secretName: {{ .SecretName }}`
164281
)
165282

166283
// dataLoaderConfig represents the configuration of the Dataloader Job.
@@ -197,7 +314,11 @@ func (factory *Factory) CreateDataLoaderIfAbsent(cluster *FdbCluster) {
197314
return
198315
}
199316

200-
t, err := template.New("dataLoaderJob").Parse(dataLoaderJob)
317+
dataLoaderJobTemplate := dataLoaderJob
318+
if factory.options.featureOperatorUnifiedImage {
319+
dataLoaderJobTemplate = dataLoaderJobUnifiedImage
320+
}
321+
t, err := template.New("dataLoaderJob").Parse(dataLoaderJobTemplate)
201322
gomega.Expect(err).NotTo(gomega.HaveOccurred())
202323
buf := bytes.Buffer{}
203324
gomega.Expect(t.Execute(&buf, factory.getDataLoaderConfig(cluster))).NotTo(gomega.HaveOccurred())

‎e2e/fixtures/fdb_operator_client.go

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,7 @@ spec:
235235
runAsGroup: 0
236236
# Install this library in a special location to force the operator to
237237
# use it as the primary library.
238-
{{ if eq .FDBVersion.Compact "7.1" }}
238+
{{ if .CopyAsPrimary }}
239239
- name: foundationdb-kubernetes-init-7-1-primary
240240
image: {{ .BaseImage }}:{{ .SidecarTag}}
241241
imagePullPolicy: {{ .ImagePullPolicy }}
@@ -383,7 +383,7 @@ spec:
383383
- fdbrestore
384384
- --copy-library
385385
- "{{ .FDBVersion.Compact }}"
386-
{{ if eq .FDBVersion.Compact "7.1" }}
386+
{{ if .CopyAsPrimary }}
387387
- --copy-primary-library
388388
- "{{ .FDBVersion.Compact }}"
389389
{{ end }}
@@ -509,11 +509,14 @@ type SidecarConfig struct {
509509
FDBVersion fdbv1beta2.Version
510510
// ImagePullPolicy represents the pull policy for the sidecar.
511511
ImagePullPolicy corev1.PullPolicy
512+
// CopyAsPrimary if true the version should be copied as primary library.
513+
CopyAsPrimary bool
512514
}
513515

514516
// GetSidecarConfigs returns the sidecar configs. The sidecar configs can be used to template applications that will use
515517
// all provided sidecar versions to inject FDB client libraries.
516518
func (factory *Factory) GetSidecarConfigs() []SidecarConfig {
519+
var hasCopyPrimarySet bool
517520
additionalSidecarVersions := factory.GetAdditionalSidecarVersions()
518521
sidecarConfigs := make([]SidecarConfig, 0, len(additionalSidecarVersions)+1)
519522

@@ -522,14 +525,21 @@ func (factory *Factory) GetSidecarConfigs() []SidecarConfig {
522525
image = factory.GetFoundationDBImage()
523526
}
524527

528+
defaultConfig := getDefaultSidecarConfig(
529+
image,
530+
factory.GetFDBVersion(),
531+
factory.getImagePullPolicy(),
532+
factory.options.featureOperatorUnifiedImage,
533+
)
534+
535+
if factory.GetFDBVersion().SupportsDNSInClusterFile() {
536+
defaultConfig.CopyAsPrimary = true
537+
hasCopyPrimarySet = true
538+
}
539+
525540
sidecarConfigs = append(
526541
sidecarConfigs,
527-
getDefaultSidecarConfig(
528-
image,
529-
factory.GetFDBVersion(),
530-
factory.getImagePullPolicy(),
531-
factory.options.featureOperatorUnifiedImage,
532-
),
542+
defaultConfig,
533543
)
534544
baseImage := sidecarConfigs[0].BaseImage
535545

@@ -540,9 +550,15 @@ func (factory *Factory) GetSidecarConfigs() []SidecarConfig {
540550
continue
541551
}
542552

553+
sidecarConfig := getSidecarConfig(baseImage, "", version, factory.getImagePullPolicy(), factory.options.featureOperatorUnifiedImage)
554+
if !hasCopyPrimarySet && version.SupportsDNSInClusterFile() {
555+
sidecarConfig.CopyAsPrimary = true
556+
hasCopyPrimarySet = true
557+
}
558+
543559
sidecarConfigs = append(
544560
sidecarConfigs,
545-
getSidecarConfig(baseImage, "", version, factory.getImagePullPolicy(), factory.options.featureOperatorUnifiedImage),
561+
sidecarConfig,
546562
)
547563
}
548564

‎e2e/test_operator/operator_test.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ var _ = BeforeSuite(func() {
8181
)
8282

8383
//Load some data async into the cluster. We will only block as long as the Job is created.
84+
// TODO (johscheuer): Fix this method for the unified image
8485
factory.CreateDataLoaderIfAbsent(fdbCluster)
8586

8687
// In order to test the robustness of the operator we try to kill the operator Pods every minute.

0 commit comments

Comments
 (0)