Skip to content

Commit 80c8cd1

Browse files
authored
Chore: Add hardcoded-records source to profiling script (#339)
1 parent f05baf1 commit 80c8cd1

File tree

1 file changed

+23
-3
lines changed

1 file changed

+23
-3
lines changed

examples/run_perf_test_reads.py

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,16 @@
4949
poetry run python ./examples/run_perf_test_reads.py -e=5 --destination=e2e --no-cache
5050
```
5151
52+
Testing Python CDK throughput:
53+
54+
```bash
55+
# Test max throughput:
56+
poetry run python ./examples/run_perf_test_reads.py -n=2400000 --source=hardcoded --destination=e2e
57+
# Analyze tracing data:
58+
poetry run viztracer --open -- ./examples/run_perf_test_reads.py -e=3 --source=hardcoded --destination=e2e
59+
```
60+
61+
5262
Note:
5363
- The Faker stream ('purchases') is assumed to be 220 bytes, meaning 4_500 records is
5464
approximately 1 MB. Based on this: 25K records/second is approximately 5.5 MB/s.
@@ -157,6 +167,15 @@ def get_source(
157167
},
158168
)
159169

170+
if source_alias == "hardcoded":
171+
return ab.get_source(
172+
"source-hardcoded-records",
173+
streams=["dummy_fields"],
174+
config={
175+
"count": num_records,
176+
},
177+
)
178+
160179
raise ValueError(f"Unknown source alias: {source_alias}") # noqa: TRY003
161180

162181

@@ -244,10 +263,11 @@ def main(
244263
type=str,
245264
help=(
246265
"The cache type to use. The `e2e` source is recommended when Docker is available, "
247-
"while the `faker` source runs natively in Python."
266+
"while the `faker` source runs natively in Python. The 'hardcoded' source is "
267+
"similar to the 'e2e' source, but written in Python."
248268
),
249-
choices=["faker", "e2e"],
250-
default="e2e",
269+
choices=["faker", "e2e", "hardcoded"],
270+
default="hardcoded",
251271
)
252272
parser.add_argument(
253273
"--destination",

0 commit comments

Comments
 (0)