Skip to content

Commit 1a0e3c7

Browse files
tim-quixtim-quix
authored
add tsbs as source (#613)
* add tsbs * tweak categories * more tweaks to library.json * alter deployment options * rename folder for binaries * adjust file permissions * convert to using repo and compiling directly * fix bad variable
1 parent e6d2038 commit 1a0e3c7

File tree

5 files changed

+149
-0
lines changed

5 files changed

+149
-0
lines changed

‎docker/sources/tsbs/README.md‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# TSBS Quix Data Generator
2+
3+
Using the [Time Series Benchmark Suite (TSBS)](https://github.com/timescale/tsbs) by Timescale,
4+
this generates a set of data for doing benchmarks with.
5+
6+
It rolls up the `tsbs_generate_data` and `tsbs_load_*` operations into one simple
7+
operation, producing data to a specified Quix Cloud topic.
8+
9+
## How to Use
10+
11+
First, we recommend checking out the [TSBS README](https://github.com/timescale/tsbs),
12+
as this simply extends it to produce data to a Kafka topic.
13+
14+
Just provide the necessary environment variables, which will
15+
configure how the data is generated, and what topic the data is dumped to.
16+
17+
18+
19+
20+
21+

‎docker/sources/tsbs/dockerfile‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Stage 1: build Go runtime binaries
2+
FROM golang:1.20 AS builder
3+
RUN apt-get update && apt-get install -y git make
4+
RUN git clone --depth 1 --branch quix https://github.com/quixio/tsbs-quix.git /src
5+
WORKDIR /src
6+
RUN make tsbs_generate_data tsbs_load_kafka_quix
7+
8+
9+
# Stage 2: build runtime container
10+
FROM debian:bookworm-slim
11+
12+
# Install CA certs for HTTPS
13+
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && \
14+
apt-get clean && rm -rf /var/lib/apt/lists/*
15+
16+
COPY --from=builder /src/bin/tsbs_generate_data /app/bin/tsbs_generate_data
17+
COPY --from=builder /src/bin/tsbs_load_kafka_quix /app/bin/tsbs_load_kafka_quix
18+
COPY ./main.sh /app/main.sh
19+
WORKDIR /app
20+
21+
RUN chmod +x main.sh bin/tsbs_generate_data bin/tsbs_load_kafka_quix
22+
23+
ENTRYPOINT ["./main.sh"]

‎docker/sources/tsbs/icon.png‎

44.9 KB
Loading

‎docker/sources/tsbs/library.json‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
{
2+
"libraryItemId": "tsbs-quix-data-generator",
3+
"name": "TSBS Quix Data Generator",
4+
"language": "Go",
5+
"IconFile": "icon.png",
6+
"tags": {
7+
"Pipeline Stage": ["Source"],
8+
"Type": ["Connectors"],
9+
"Category": ["External APIs"],
10+
"Technology": ["TSBS"]
11+
},
12+
"shortDescription": "Push sample data to a Quix topic using TSBS data generation",
13+
"longDescription": "Push sample data to a Quix topic using TSBS data generation",
14+
"DefaultFile": "main.sh",
15+
"EntryPoint": "dockerfile",
16+
"RunEntryPoint": "main.sh",
17+
"Variables": [
18+
{
19+
"Name": "output",
20+
"Type": "EnvironmentVariable",
21+
"InputType": "OutputTopic",
22+
"Description": "This is the Quix Topic that will receive the generated data",
23+
"DefaultValue": "tsbs_data",
24+
"Required": true
25+
},
26+
{
27+
"Name": "TSBS_USE_CASE",
28+
"Type": "EnvironmentVariable",
29+
"InputType": "FreeText",
30+
"Description": "The type of data; options: cpu-only, devops, iot",
31+
"DefaultValue": "cpu-only",
32+
"Required": true
33+
},
34+
{
35+
"Name": "TSBS_SEED",
36+
"Type": "EnvironmentVariable",
37+
"InputType": "FreeText",
38+
"Description": "an integer PRNG seed for deterministic generation. ",
39+
"DefaultValue": "123",
40+
"Required": true
41+
},
42+
{
43+
"Name": "TSBS_SCALE",
44+
"Type": "EnvironmentVariable",
45+
"InputType": "FreeText",
46+
"Description": "The number of devices / trucks to generate data for",
47+
"DefaultValue": "5",
48+
"Required": true
49+
},
50+
{
51+
"Name": "TSBS_TIMESTAMP_START",
52+
"Type": "EnvironmentVariable",
53+
"InputType": "FreeText",
54+
"Description": "A start time for the data's timestamps, formatted as Y-M-DTH:M:SZ",
55+
"DefaultValue": "2025-01-01T00:00:00Z",
56+
"Required": true
57+
},
58+
{
59+
"Name": "TSBS_TIMESTAMP_END",
60+
"Type": "EnvironmentVariable",
61+
"InputType": "FreeText",
62+
"Description": "The end time for the data's timestamps, formatted as Y-M-DTH:M:SZ",
63+
"DefaultValue": "2025-01-01T00:10:00Z",
64+
"Required": true
65+
},
66+
{
67+
"Name": "TSBS_LOG_INTERVAL",
68+
"Type": "EnvironmentVariable",
69+
"InputType": "FreeText",
70+
"Description": "how much time should be between each reading per device, formatted as Nt",
71+
"DefaultValue": "10s",
72+
"Required": true
73+
}
74+
],
75+
"DeploySettings": {
76+
"DeploymentType": "Job",
77+
"CpuMillicores": 200,
78+
"MemoryInMb": 1000,
79+
"Replicas": 1,
80+
"PublicAccess": false,
81+
"ValidateConnection": false
82+
}
83+
}

‎docker/sources/tsbs/main.sh‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/bin/sh
2+
set -e
3+
4+
GEN_ARGS="
5+
--format=jsonlines
6+
--use-case=${TSBS_USE_CASE:-cpu-only}
7+
--seed=${TSBS_SEED:-123}
8+
--scale=${TSBS_SCALE:-5}
9+
--timestamp-start=${TSBS_TIMESTAMP_START:-2025-01-01T00:00:00Z}
10+
--timestamp-end=${TSBS_TIMESTAMP_END:-2025-01-01T00:10:00Z}
11+
--log-interval=${TSBS_LOG_INTERVAL:-10s}
12+
"
13+
14+
LOAD_ARGS="
15+
--api-url=${Quix__Portal__Api:-https://portal-api.platform.quix.io/}
16+
--workspace=${Quix__Workspace__Id:?Must set Quix__Workspace__Id}
17+
--token=${Quix__Sdk__Token:?Must set Quix__Sdk__Token}
18+
--topic=${output:-tsbs_data}
19+
"
20+
21+
# Execute the pipeline
22+
exec ./bin/tsbs_generate_data $GEN_ARGS | ./bin/tsbs_load_kafka_quix $LOAD_ARGS

0 commit comments

Comments
 (0)