Skip to content

Commit 8876aa8

Browse files
johscheuerjohscheuer
committed
Rewrite the data-loader in go and allow the data-laoder to be modified
1 parent 3b93606 commit 8876aa8

File tree

9 files changed

+223
-93
lines changed

9 files changed

+223
-93
lines changed

‎Dockerfile

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -52,13 +52,6 @@ COPY kubectl-fdb/ kubectl-fdb/
5252
# Build
5353
RUN CGO_ENABLED=1 GOOS=linux GOARCH=${TARGETARCH} GO111MODULE=on make manager plugin-go
5454

55-
# Create user and group here since we don't have the tools
56-
# in distroless
57-
RUN groupadd --gid 4059 fdb && \
58-
useradd --gid 4059 --uid 4059 --create-home --shell /bin/bash fdb && \
59-
mkdir -p /var/log/fdb && \
60-
touch /var/log/fdb/.keep
61-
6255
FROM docker.io/rockylinux/rockylinux:9.5-minimal
6356

6457
ARG FDB_VERSION
@@ -95,11 +88,15 @@ RUN set -eux && \
9588
rpm -i foundationdb-clients-${FDB_VERSION}-1.${FDB_OS}.${FDB_ARCH}.rpm --excludepath=/usr/bin --excludepath=/usr/lib/foundationdb/backup_agent && \
9689
rm foundationdb-clients-${FDB_VERSION}-1.${FDB_OS}.${FDB_ARCH}.rpm foundationdb-clients-${FDB_VERSION}-1.${FDB_OS}.${FDB_ARCH}.rpm.sha256
9790

98-
COPY --from=builder /etc/passwd /etc/passwd
99-
COPY --from=builder /etc/group /etc/group
91+
# Create user and group here since we don't have the tools
92+
# in distroless
93+
RUN groupadd --gid 4059 fdb && \
94+
useradd --gid 4059 --uid 4059 --create-home --shell /bin/bash fdb && \
95+
mkdir -p /var/log/fdb && \
96+
touch /var/log/fdb/.keep \
97+
10098
COPY --chown=fdb:fdb --from=builder /workspace/bin/manager .
10199
COPY --chown=fdb:fdb --from=builder /workspace/bin/kubectl-fdb /usr/local/bin/kubectl-fdb
102-
COPY --chown=fdb:fdb --from=builder /var/log/fdb/.keep /var/log/fdb/.keep
103100

104101
# Set to the numeric UID of fdb user to satisfy PodSecurityPolices which enforce runAsNonRoot
105102
USER 4059

‎e2e/fixtures/fdb_data_loader.go

Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
"time"
3131

3232
k8serrors "k8s.io/apimachinery/pkg/api/errors"
33+
"k8s.io/utils/pointer"
3334

3435
"github.com/onsi/gomega"
3536
batchv1 "k8s.io/api/batch/v1"
@@ -66,7 +67,7 @@ spec:
6667
name: {{ .Name }}
6768
# This configuration will load ~1GB per data loader.
6869
args:
69-
- --keys=1000000
70+
- --keys={{ .Keys }}
7071
- --batch-size=50
7172
- --value-size=1000
7273
env:
@@ -86,6 +87,8 @@ spec:
8687
value: /var/dynamic/fdb/primary/lib
8788
- name: FDB_NETWORK_OPTION_TRACE_LOG_GROUP
8889
value: {{ .Name }}
90+
- name: FDB_NETWORK_OPTION_TRACE_ENABLE
91+
value: "/tmp/fdb-trace-logs"
8992
- name: FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY
9093
value: /var/dynamic/fdb/libs
9194
- name: PYTHONUNBUFFERED
@@ -98,6 +101,8 @@ spec:
98101
- name: fdb-certs
99102
mountPath: /tmp/fdb-certs
100103
readOnly: true
104+
- name: fdb-logs
105+
mountPath: /tmp/fdb-trace-logs
101106
resources:
102107
requests:
103108
cpu: "1"
@@ -162,11 +167,13 @@ spec:
162167
path: fdb.cluster
163168
- name: fdb-libs
164169
emptyDir: {}
170+
- name: fdb-logs
171+
emptyDir: {}
165172
- name: fdb-certs
166173
secret:
167174
secretName: {{ .SecretName }}`
168175

169-
// For now we only load 2GB into the cluster, we can increase this later if we want.
176+
// For now, we only load 2GB into the cluster, we can increase this later if we want.
170177
dataLoaderJobUnifiedImage = `apiVersion: batch/v1
171178
kind: Job
172179
metadata:
@@ -186,7 +193,7 @@ spec:
186193
name: {{ .Name }}
187194
# This configuration will load ~1GB per data loader.
188195
args:
189-
- --keys=1000000
196+
- --keys={{ .Keys }}
190197
- --batch-size=50
191198
- --value-size=1000
192199
env:
@@ -202,6 +209,8 @@ spec:
202209
# Consider remove this option once 6.3 is no longer being used.
203210
- name: FDB_NETWORK_OPTION_IGNORE_EXTERNAL_CLIENT_FAILURES
204211
value: ""
212+
- name: FDB_NETWORK_OPTION_TRACE_ENABLE
213+
value: "/tmp/fdb-trace-logs"
205214
- name: LD_LIBRARY_PATH
206215
value: /var/dynamic/fdb
207216
- name: FDB_NETWORK_OPTION_TRACE_LOG_GROUP
@@ -218,6 +227,8 @@ spec:
218227
- name: fdb-certs
219228
mountPath: /tmp/fdb-certs
220229
readOnly: true
230+
- name: fdb-logs
231+
mountPath: /tmp/fdb-trace-logs
221232
resources:
222233
requests:
223234
cpu: "1"
@@ -279,12 +290,14 @@ spec:
279290
path: fdb.cluster
280291
- name: fdb-libs
281292
emptyDir: {}
293+
- name: fdb-logs
294+
emptyDir: {}
282295
- name: fdb-certs
283296
secret:
284297
secretName: {{ .SecretName }}`
285298
)
286299

287-
// dataLoaderConfig represents the configuration of the Dataloader Job.
300+
// dataLoaderConfig represents the configuration of the data-loader Job.
288301
type dataLoaderConfig struct {
289302
// Name of the data loader Job.
290303
Name string
@@ -299,27 +312,31 @@ type dataLoaderConfig struct {
299312
// SecretName represents the Kubernetes secret that contains the certificates for communicating with the FoundationDB
300313
// cluster.
301314
SecretName string
315+
// Keys defines how many keys should be written by the data loader.
316+
Keys int
302317
}
303318

304-
func (factory *Factory) getDataLoaderConfig(cluster *FdbCluster) *dataLoaderConfig {
319+
func (factory *Factory) getDataLoaderConfig(cluster *FdbCluster, keys *int) *dataLoaderConfig {
320+
log.Println("keys:", keys, "deref:", pointer.IntDeref(keys, 1000000))
305321
return &dataLoaderConfig{
306322
Name: dataLoaderName,
307323
Image: factory.GetDataLoaderImage(),
308324
Namespace: cluster.Namespace(),
309325
SidecarVersions: factory.GetSidecarConfigs(),
310326
ClusterName: cluster.Name(),
311327
SecretName: factory.GetSecretName(),
328+
Keys: pointer.IntDeref(keys, 1000000),
312329
}
313330
}
314331

315332
// CreateDataLoaderIfAbsent will create the data loader for the provided cluster and load some random data into the cluster.
316333
func (factory *Factory) CreateDataLoaderIfAbsent(cluster *FdbCluster) {
317-
factory.CreateDataLoaderIfAbsentWithWait(cluster, true)
334+
factory.CreateDataLoaderIfAbsentWithWait(cluster, nil, true)
318335
}
319336

320337
// CreateDataLoaderIfAbsentWithWait will create the data loader for the provided cluster and load some random data into the cluster.
321338
// If wait is true, the method will wait until the data loader has finished.
322-
func (factory *Factory) CreateDataLoaderIfAbsentWithWait(cluster *FdbCluster, wait bool) {
339+
func (factory *Factory) CreateDataLoaderIfAbsentWithWait(cluster *FdbCluster, keys *int, wait bool) {
323340
if !factory.options.enableDataLoading {
324341
return
325342
}
@@ -331,7 +348,8 @@ func (factory *Factory) CreateDataLoaderIfAbsentWithWait(cluster *FdbCluster, wa
331348
t, err := template.New("dataLoaderJob").Parse(dataLoaderJobTemplate)
332349
gomega.Expect(err).NotTo(gomega.HaveOccurred())
333350
buf := bytes.Buffer{}
334-
gomega.Expect(t.Execute(&buf, factory.getDataLoaderConfig(cluster))).NotTo(gomega.HaveOccurred())
351+
log.Println("keys:", keys)
352+
gomega.Expect(t.Execute(&buf, factory.getDataLoaderConfig(cluster, keys))).NotTo(gomega.HaveOccurred())
335353
decoder := yamlutil.NewYAMLOrJSONDecoder(&buf, 100000)
336354
for {
337355
var rawObj runtime.RawExtension

‎e2e/test_operator_ha/operator_ha_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -337,7 +337,7 @@ var _ = Describe("Operator HA tests", Label("e2e", "pr"), func() {
337337
})
338338

339339
// TODO (johscheuer): Allow to have this as a long running task until the test is done.
340-
factory.CreateDataLoaderIfAbsentWithWait(fdbCluster.GetPrimary(), false)
340+
factory.CreateDataLoaderIfAbsentWithWait(fdbCluster.GetPrimary(), nil, false)
341341

342342
time.Sleep(1 * time.Minute)
343343
log.Println("replacedPod", replacedPod.Name, "useLocalitiesForExclusion", fdbCluster.GetPrimary().GetCluster().UseLocalitiesForExclusion())
@@ -454,7 +454,7 @@ var _ = Describe("Operator HA tests", Label("e2e", "pr"), func() {
454454
})
455455

456456
// TODO (johscheuer): Allow to have this as a long running task until the test is done.
457-
factory.CreateDataLoaderIfAbsentWithWait(fdbCluster.GetPrimary(), false)
457+
factory.CreateDataLoaderIfAbsentWithWait(fdbCluster.GetPrimary(), nil, false)
458458

459459
time.Sleep(1 * time.Minute)
460460
log.Println("replacedPod", replacedPod.Name, "useLocalitiesForExclusion", fdbCluster.GetPrimary().GetCluster().UseLocalitiesForExclusion())

‎sample-apps/data-loader/Dockerfile

Lines changed: 75 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,83 @@
1-
FROM docker.io/library/python:3.13-slim
1+
ARG FDB_VERSION=7.1.67
2+
ARG FDB_WEBSITE=https://github.com/apple/foundationdb/releases/download
23

3-
ARG TARGETARCH
4+
# Build the manager binary
5+
FROM docker.io/library/golang:1.24.4 AS builder
46

5-
COPY app.py /usr/local/bin
7+
ARG FDB_VERSION
8+
ARG FDB_WEBSITE
9+
ARG TAG="latest"
610

7-
RUN pip install foundationdb==7.1.67
8-
RUN groupadd --gid 4059 fdb && \
9-
useradd --gid 4059 --uid 4059 --shell /usr/sbin/nologin fdb
11+
RUN set -eux && \
12+
if [ "$TARGETARCH" = "amd64" ]; then \
13+
FDB_ARCH=amd64; \
14+
elif [ "$TARGETARCH" = "arm64" ]; then \
15+
FDB_ARCH=aarch64; \
16+
if [ "${FDB_VERSION%.*}" = "7.1" ]; then \
17+
FDB_VERSION="7.3.63"; \
18+
fi; \
19+
else \
20+
echo "ERROR: unsupported architecture $TARGETARCH" 1>&2; \
21+
exit 1; \
22+
fi; \
23+
curl --fail -L "${FDB_WEBSITE}/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH}.deb" -o foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH}.deb && \
24+
curl --fail -L "${FDB_WEBSITE}/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH}.deb.sha256" -o foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH}.deb.sha256 && \
25+
sha256sum -c foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH}.deb.sha256 && \
26+
dpkg -i foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH}.deb && \
27+
rm foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH}.deb foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH}.deb.sha256
1028

11-
RUN apt-get update && \
12-
apt-get install -y --no-install-recommends curl && \
13-
curl -L https://github.com/krallin/tini/releases/download/v0.19.0/tini-${TARGETARCH} -o tini-${TARGETARCH} && \
14-
echo "93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c tini-amd64\n07952557df20bfd2a95f9bef198b445e006171969499a1d361bd9e6f8e5e0e81 tini-arm64" > tini-sha.txt && \
15-
sha256sum --quiet --ignore-missing -c tini-sha.txt && \
16-
chmod +x tini-${TARGETARCH} && \
17-
mv tini-${TARGETARCH} /usr/bin/tini && \
18-
rm -rf /tmp/*
29+
WORKDIR /workspace
30+
# Copy the Go Modules manifests
31+
COPY go.mod go.mod
32+
COPY go.sum go.sum
33+
# cache deps before building and copying source so that we don't need to re-download as much
34+
# and so that source changes don't invalidate our downloaded layer
35+
RUN go mod download -x
36+
37+
# Copy the go source
38+
COPY main.go main.go
39+
40+
# Build
41+
RUN CGO_ENABLED=1 GOOS=linux GOARCH=${TARGETARCH} GO111MODULE=on go build -o /workspace/bin/data-loader main.go
42+
43+
FROM docker.io/rockylinux/rockylinux:9.5-minimal
44+
45+
ARG FDB_VERSION
46+
ARG FDB_WEBSITE
47+
48+
VOLUME /usr/lib/fdb
49+
50+
WORKDIR /
51+
52+
RUN set -eux && \
53+
if [ "$TARGETARCH" = "amd64" ]; then \
54+
FDB_ARCH=x86_64; \
55+
elif [ "$TARGETARCH" = "arm64" ]; then \
56+
FDB_ARCH=aarch64; \
57+
if [ "${FDB_VERSION%.*}" = "7.1" ]; then \
58+
FDB_VERSION="7.3.63"; \
59+
fi; \
60+
else \
61+
echo "ERROR: unsupported architecture $TARGETARCH" 1>&2; \
62+
exit 1; \
63+
fi; \
64+
if [ "${FDB_VERSION%.*}" = "7.1" ]; then \
65+
# FDB 7.1 published the client packages for el7, 7.3 and newer uses el9.
66+
FDB_OS=el7; \
67+
else \
68+
FDB_OS=el9; \
69+
fi; \
70+
curl --fail -L "${FDB_WEBSITE}/${FDB_VERSION}/foundationdb-clients-${FDB_VERSION}-1.${FDB_OS}.${FDB_ARCH}.rpm" -o foundationdb-clients-${FDB_VERSION}-1.${FDB_OS}.${FDB_ARCH}.rpm && \
71+
curl --fail -L "${FDB_WEBSITE}/${FDB_VERSION}/foundationdb-clients-${FDB_VERSION}-1.${FDB_OS}.${FDB_ARCH}.rpm.sha256" -o foundationdb-clients-${FDB_VERSION}-1.${FDB_OS}.${FDB_ARCH}.rpm.sha256 && \
72+
microdnf install -y glibc pkg-config && \
73+
microdnf clean all && \
74+
sha256sum -c foundationdb-clients-${FDB_VERSION}-1.${FDB_OS}.${FDB_ARCH}.rpm.sha256 && \
75+
rpm -i foundationdb-clients-${FDB_VERSION}-1.${FDB_OS}.${FDB_ARCH}.rpm --excludepath=/usr/bin --excludepath=/usr/lib/foundationdb/backup_agent && \
76+
rm foundationdb-clients-${FDB_VERSION}-1.${FDB_OS}.${FDB_ARCH}.rpm foundationdb-clients-${FDB_VERSION}-1.${FDB_OS}.${FDB_ARCH}.rpm.sha256
77+
78+
COPY --chown=fdb:fdb --from=builder /workspace/bin/data-loader /usr/local/bin/data-loader
1979

2080
# Set to the numeric UID of fdb user to satisfy PodSecurityPolices which enforce runAsNonRoot
2181
USER 4059
2282

23-
ENTRYPOINT [ "/usr/bin/tini", "-g", "--", "python", "/usr/local/bin/app.py" ]
83+
ENTRYPOINT ["/usr/local/bin/data-loader"]

‎sample-apps/data-loader/app.py

Lines changed: 0 additions & 54 deletions
This file was deleted.

‎sample-apps/data-loader/go.mod

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
module github.com/FoundationDB/fdb-data-loader
2+
3+
go 1.22.7
4+
5+
require (
6+
// Binding version for 7.1.67
7+
github.com/apple/foundationdb/bindings/go v0.0.0-20250115161953-f1ab8147ed1c
8+
github.com/google/uuid v1.6.0
9+
)

‎sample-apps/data-loader/go.sum

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
github.com/apple/foundationdb/bindings/go v0.0.0-20250115161953-f1ab8147ed1c h1:Nnun3T50beIpO6YKDZInHuZMgnNtYJafSlQAvkXwOWc=
2+
github.com/apple/foundationdb/bindings/go v0.0.0-20250115161953-f1ab8147ed1c/go.mod h1:OMVSB21p9+xQUIqlGizHPZfjK+SHws1ht+ZytVDoz9U=
3+
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
4+
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=

‎sample-apps/data-loader/job.yaml

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,15 +24,20 @@ spec:
2424
mountPath: /var/dynamic-conf
2525
initContainers:
2626
- name: foundationdb-kubernetes-init
27-
image: foundationdb/foundationdb-kubernetes-sidecar:6.3.3-1
27+
image: foundationdb/fdb-kubernetes-monitor:7.1.57
2828
args:
2929
- "--copy-file"
3030
- "fdb.cluster"
3131
- "--copy-library"
32-
- "6.3"
32+
- "7.1"
3333
- "--copy-library"
34-
- "6.2"
35-
- "--init-mode"
34+
- "6.3"
35+
- "--mode"
36+
- "init"
37+
- "--input-dir"
38+
- "/var/dynamic-conf"
39+
- "--output-dir"
40+
- "/var/output-files"
3641
- "--require-not-empty"
3742
- "fdb.cluster"
3843
volumeMounts:

0 commit comments

Comments
 (0)