Skip to content

Commit ce6dbeb

Browse files
johscheuerjohscheuer
committed
Rewrite the data-loader in go and allow the data-laoder to be modified
1 parent 14e271f commit ce6dbeb

File tree

10 files changed

+199
-94
lines changed

10 files changed

+199
-94
lines changed

‎Dockerfile

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -38,13 +38,6 @@ COPY kubectl-fdb/ kubectl-fdb/
3838
# Build
3939
RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on make manager plugin-go
4040

41-
# Create user and group here since we don't have the tools
42-
# in distroless
43-
RUN groupadd --gid 4059 fdb && \
44-
useradd --gid 4059 --uid 4059 --create-home --shell /bin/bash fdb && \
45-
mkdir -p /var/log/fdb && \
46-
touch /var/log/fdb/.keep
47-
4841
FROM docker.io/rockylinux/rockylinux:9.5-minimal
4942

5043
ARG FDB_VERSION
@@ -63,11 +56,15 @@ RUN set -eux && \
6356
rpm -i foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm --excludepath=/usr/bin --excludepath=/usr/lib/foundationdb/backup_agent && \
6457
rm foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm.sha256
6558

66-
COPY --from=builder /etc/passwd /etc/passwd
67-
COPY --from=builder /etc/group /etc/group
59+
# Create user and group here since we don't have the tools
60+
# in distroless
61+
RUN groupadd --gid 4059 fdb && \
62+
useradd --gid 4059 --uid 4059 --create-home --shell /bin/bash fdb && \
63+
mkdir -p /var/log/fdb && \
64+
touch /var/log/fdb/.keep \
65+
6866
COPY --chown=fdb:fdb --from=builder /workspace/bin/manager .
6967
COPY --chown=fdb:fdb --from=builder /workspace/bin/kubectl-fdb /usr/local/bin/kubectl-fdb
70-
COPY --chown=fdb:fdb --from=builder /var/log/fdb/.keep /var/log/fdb/.keep
7168

7269
# Set to the numeric UID of fdb user to satisfy PodSecurityPolices which enforce runAsNonRoot
7370
USER 4059

‎e2e/fixtures/fdb_data_loader.go

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,13 @@ import (
2525
"context"
2626
"errors"
2727
"io"
28-
k8serrors "k8s.io/apimachinery/pkg/api/errors"
2928
"log"
3029
"text/template"
3130
"time"
3231

32+
k8serrors "k8s.io/apimachinery/pkg/api/errors"
33+
"k8s.io/utils/pointer"
34+
3335
"github.com/onsi/gomega"
3436
batchv1 "k8s.io/api/batch/v1"
3537
corev1 "k8s.io/api/core/v1"
@@ -65,7 +67,7 @@ spec:
6567
name: {{ .Name }}
6668
# This configuration will load ~1GB per data loader.
6769
args:
68-
- --keys=1000000
70+
- --keys={{ .Keys }}
6971
- --batch-size=50
7072
- --value-size=1000
7173
env:
@@ -85,6 +87,8 @@ spec:
8587
value: /var/dynamic/fdb/primary/lib
8688
- name: FDB_NETWORK_OPTION_TRACE_LOG_GROUP
8789
value: {{ .Name }}
90+
- name: FDB_NETWORK_OPTION_TRACE_ENABLE
91+
value: "/tmp/fdb-trace-logs"
8892
- name: FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY
8993
value: /var/dynamic/fdb/libs
9094
- name: PYTHONUNBUFFERED
@@ -97,6 +101,8 @@ spec:
97101
- name: fdb-certs
98102
mountPath: /tmp/fdb-certs
99103
readOnly: true
104+
- name: fdb-logs
105+
mountPath: /tmp/fdb-trace-logs
100106
resources:
101107
requests:
102108
cpu: "1"
@@ -161,11 +167,13 @@ spec:
161167
path: fdb.cluster
162168
- name: fdb-libs
163169
emptyDir: {}
170+
- name: fdb-logs
171+
emptyDir: {}
164172
- name: fdb-certs
165173
secret:
166174
secretName: {{ .SecretName }}`
167175

168-
// For now we only load 2GB into the cluster, we can increase this later if we want.
176+
// For now, we only load 2GB into the cluster, we can increase this later if we want.
169177
dataLoaderJobUnifiedImage = `apiVersion: batch/v1
170178
kind: Job
171179
metadata:
@@ -185,7 +193,7 @@ spec:
185193
name: {{ .Name }}
186194
# This configuration will load ~1GB per data loader.
187195
args:
188-
- --keys=1000000
196+
- --keys={{ .Keys }}
189197
- --batch-size=50
190198
- --value-size=1000
191199
env:
@@ -201,6 +209,8 @@ spec:
201209
# Consider remove this option once 6.3 is no longer being used.
202210
- name: FDB_NETWORK_OPTION_IGNORE_EXTERNAL_CLIENT_FAILURES
203211
value: ""
212+
- name: FDB_NETWORK_OPTION_TRACE_ENABLE
213+
value: "/tmp/fdb-trace-logs"
204214
- name: LD_LIBRARY_PATH
205215
value: /var/dynamic/fdb
206216
- name: FDB_NETWORK_OPTION_TRACE_LOG_GROUP
@@ -217,6 +227,8 @@ spec:
217227
- name: fdb-certs
218228
mountPath: /tmp/fdb-certs
219229
readOnly: true
230+
- name: fdb-logs
231+
mountPath: /tmp/fdb-trace-logs
220232
resources:
221233
requests:
222234
cpu: "1"
@@ -278,12 +290,14 @@ spec:
278290
path: fdb.cluster
279291
- name: fdb-libs
280292
emptyDir: {}
293+
- name: fdb-logs
294+
emptyDir: {}
281295
- name: fdb-certs
282296
secret:
283297
secretName: {{ .SecretName }}`
284298
)
285299

286-
// dataLoaderConfig represents the configuration of the Dataloader Job.
300+
// dataLoaderConfig represents the configuration of the data-loader Job.
287301
type dataLoaderConfig struct {
288302
// Name of the data loader Job.
289303
Name string
@@ -298,27 +312,31 @@ type dataLoaderConfig struct {
298312
// SecretName represents the Kubernetes secret that contains the certificates for communicating with the FoundationDB
299313
// cluster.
300314
SecretName string
315+
// Keys defines how many keys should be written by the data loader.
316+
Keys int
301317
}
302318

303-
func (factory *Factory) getDataLoaderConfig(cluster *FdbCluster) *dataLoaderConfig {
319+
func (factory *Factory) getDataLoaderConfig(cluster *FdbCluster, keys *int) *dataLoaderConfig {
320+
log.Println("keys:", keys, "deref:", pointer.IntDeref(keys, 1000000))
304321
return &dataLoaderConfig{
305322
Name: dataLoaderName,
306323
Image: factory.GetDataLoaderImage(),
307324
Namespace: cluster.Namespace(),
308325
SidecarVersions: factory.GetSidecarConfigs(),
309326
ClusterName: cluster.Name(),
310327
SecretName: factory.GetSecretName(),
328+
Keys: pointer.IntDeref(keys, 1000000),
311329
}
312330
}
313331

314332
// CreateDataLoaderIfAbsent will create the data loader for the provided cluster and load some random data into the cluster.
315333
func (factory *Factory) CreateDataLoaderIfAbsent(cluster *FdbCluster) {
316-
factory.CreateDataLoaderIfAbsentWithWait(cluster, true)
334+
factory.CreateDataLoaderIfAbsentWithWait(cluster, nil, true)
317335
}
318336

319337
// CreateDataLoaderIfAbsentWithWait will create the data loader for the provided cluster and load some random data into the cluster.
320338
// If wait is true, the method will wait until the data loader has finished.
321-
func (factory *Factory) CreateDataLoaderIfAbsentWithWait(cluster *FdbCluster, wait bool) {
339+
func (factory *Factory) CreateDataLoaderIfAbsentWithWait(cluster *FdbCluster, keys *int, wait bool) {
322340
if !factory.options.enableDataLoading {
323341
return
324342
}
@@ -330,7 +348,8 @@ func (factory *Factory) CreateDataLoaderIfAbsentWithWait(cluster *FdbCluster, wa
330348
t, err := template.New("dataLoaderJob").Parse(dataLoaderJobTemplate)
331349
gomega.Expect(err).NotTo(gomega.HaveOccurred())
332350
buf := bytes.Buffer{}
333-
gomega.Expect(t.Execute(&buf, factory.getDataLoaderConfig(cluster))).NotTo(gomega.HaveOccurred())
351+
log.Println("keys:", keys)
352+
gomega.Expect(t.Execute(&buf, factory.getDataLoaderConfig(cluster, keys))).NotTo(gomega.HaveOccurred())
334353
decoder := yamlutil.NewYAMLOrJSONDecoder(&buf, 100000)
335354
for {
336355
var rawObj runtime.RawExtension

‎e2e/test_operator_ha/operator_ha_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -307,7 +307,7 @@ var _ = Describe("Operator HA tests", Label("e2e", "pr"), func() {
307307
})
308308

309309
// TODO (johscheuer): Allow to have this as a long running task until the test is done.
310-
factory.CreateDataLoaderIfAbsentWithWait(fdbCluster.GetPrimary(), false)
310+
factory.CreateDataLoaderIfAbsentWithWait(fdbCluster.GetPrimary(), nil, false)
311311

312312
time.Sleep(1 * time.Minute)
313313
log.Println("replacedPod", replacedPod.Name, "useLocalitiesForExclusion", fdbCluster.GetPrimary().GetCluster().UseLocalitiesForExclusion())
@@ -415,7 +415,7 @@ var _ = Describe("Operator HA tests", Label("e2e", "pr"), func() {
415415
})
416416

417417
// TODO (johscheuer): Allow to have this as a long running task until the test is done.
418-
factory.CreateDataLoaderIfAbsentWithWait(fdbCluster.GetPrimary(), false)
418+
factory.CreateDataLoaderIfAbsentWithWait(fdbCluster.GetPrimary(), nil, false)
419419

420420
time.Sleep(1 * time.Minute)
421421
log.Println("replacedPod", replacedPod.Name, "useLocalitiesForExclusion", fdbCluster.GetPrimary().GetCluster().UseLocalitiesForExclusion())

‎main.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ func init() {
4747
}
4848

4949
func main() {
50-
fdb.MustAPIVersion(620)
50+
fdb.MustAPIVersion(710)
5151
operatorOpts := setup.Options{}
5252
operatorOpts.BindFlags(flag.CommandLine)
5353

‎sample-apps/data-loader/Dockerfile

Lines changed: 48 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,55 @@
1-
FROM python:3.9-slim
1+
ARG FDB_VERSION=7.1.67
2+
ARG FDB_WEBSITE=https://github.com/apple/foundationdb/releases/download
23

3-
COPY app.py /usr/local/bin
4+
# Build the manager binary
5+
FROM docker.io/library/golang:1.23.7 AS builder
46

5-
RUN pip install foundationdb==6.2.10
6-
RUN groupadd --gid 4059 fdb && \
7-
useradd --gid 4059 --uid 4059 --shell /usr/sbin/nologin fdb
7+
ARG FDB_VERSION
8+
ARG FDB_WEBSITE
9+
ARG TAG="latest"
810

9-
RUN apt-get update && \
10-
apt-get install -y --no-install-recommends curl && \
11-
curl -Ls https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd64 -o tini && \
12-
echo "93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c tini" > tini-sha.txt && \
13-
sha256sum --quiet -c tini-sha.txt && \
14-
chmod +x tini && \
15-
mv tini /usr/bin/ && \
16-
rm -rf /tmp/*
11+
RUN set -eux && \
12+
curl --fail -L "${FDB_WEBSITE}/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_amd64.deb" -o foundationdb-clients_${FDB_VERSION}-1_amd64.deb && \
13+
curl --fail -L "${FDB_WEBSITE}/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_amd64.deb.sha256" -o foundationdb-clients_${FDB_VERSION}-1_amd64.deb.sha256 && \
14+
sha256sum -c foundationdb-clients_${FDB_VERSION}-1_amd64.deb.sha256 && \
15+
dpkg -i foundationdb-clients_${FDB_VERSION}-1_amd64.deb && \
16+
rm foundationdb-clients_${FDB_VERSION}-1_amd64.deb foundationdb-clients_${FDB_VERSION}-1_amd64.deb.sha256
17+
18+
WORKDIR /workspace
19+
# Copy the Go Modules manifests
20+
COPY go.mod go.mod
21+
COPY go.sum go.sum
22+
# cache deps before building and copying source so that we don't need to re-download as much
23+
# and so that source changes don't invalidate our downloaded layer
24+
RUN go mod download -x
25+
26+
# Copy the go source
27+
COPY main.go main.go
28+
29+
# Build
30+
RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -o /workspace/bin/data-loader main.go
31+
32+
FROM docker.io/rockylinux/rockylinux:9.5-minimal
33+
34+
ARG FDB_VERSION
35+
ARG FDB_WEBSITE
36+
37+
VOLUME /usr/lib/fdb
38+
39+
WORKDIR /
40+
41+
RUN set -eux && \
42+
curl --fail -L "${FDB_WEBSITE}/${FDB_VERSION}/foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm" -o foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm && \
43+
curl --fail -L "${FDB_WEBSITE}/${FDB_VERSION}/foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm.sha256" -o foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm.sha256 && \
44+
microdnf install -y glibc pkg-config && \
45+
microdnf clean all && \
46+
sha256sum -c foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm.sha256 && \
47+
rpm -i foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm --excludepath=/usr/bin --excludepath=/usr/lib/foundationdb/backup_agent && \
48+
rm foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm foundationdb-clients-${FDB_VERSION}-1.el7.x86_64.rpm.sha256
49+
50+
COPY --chown=fdb:fdb --from=builder /workspace/bin/data-loader /usr/local/bin/data-loader
1751

1852
# Set to the numeric UID of fdb user to satisfy PodSecurityPolices which enforce runAsNonRoot
1953
USER 4059
2054

21-
ENTRYPOINT [ "/usr/bin/tini", "-g", "--", "python", "/usr/local/bin/app.py" ]
55+
ENTRYPOINT ["/usr/local/bin/data-loader"]

‎sample-apps/data-loader/app.py

Lines changed: 0 additions & 54 deletions
This file was deleted.

‎sample-apps/data-loader/go.mod

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
module github.com/FoundationDB/fdb-data-loader
2+
3+
go 1.22.7
4+
5+
require (
6+
// Binding version for 7.1.67
7+
github.com/apple/foundationdb/bindings/go v0.0.0-20250115161953-f1ab8147ed1c
8+
github.com/google/uuid v1.6.0
9+
)

‎sample-apps/data-loader/go.sum

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
github.com/apple/foundationdb/bindings/go v0.0.0-20250115161953-f1ab8147ed1c h1:Nnun3T50beIpO6YKDZInHuZMgnNtYJafSlQAvkXwOWc=
2+
github.com/apple/foundationdb/bindings/go v0.0.0-20250115161953-f1ab8147ed1c/go.mod h1:OMVSB21p9+xQUIqlGizHPZfjK+SHws1ht+ZytVDoz9U=
3+
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
4+
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=

‎sample-apps/data-loader/job.yaml

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,15 +24,20 @@ spec:
2424
mountPath: /var/dynamic-conf
2525
initContainers:
2626
- name: foundationdb-kubernetes-init
27-
image: foundationdb/foundationdb-kubernetes-sidecar:6.3.3-1
27+
image: foundationdb/fdb-kubernetes-monitor:7.1.57
2828
args:
2929
- "--copy-file"
3030
- "fdb.cluster"
3131
- "--copy-library"
32-
- "6.3"
32+
- "7.1"
3333
- "--copy-library"
34-
- "6.2"
35-
- "--init-mode"
34+
- "6.3"
35+
- "--mode"
36+
- "init"
37+
- "--input-dir"
38+
- "/var/dynamic-conf"
39+
- "--output-dir"
40+
- "/var/output-files"
3641
- "--require-not-empty"
3742
- "fdb.cluster"
3843
volumeMounts:

0 commit comments

Comments
 (0)