Skip to content

Commit a697414

Browse files
freya022freya022
committed
Rework the test data generator as a Gradle task
Added support for shards, chunks now include all compression variants
1 parent a79f1e9 commit a697414

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+345
-311
lines changed

‎.gitignore‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,5 @@ build
66
.idea
77

88
# Secret key and algorithm parameters for test data decryption
9-
**/chunks-*/Key.bin
10-
**/chunks-*/Parameters.bin
9+
**/chunks/Key.bin
10+
**/chunks/Parameters.bin

‎benchmarks/src/jmh/java/dev/freya02/discord/zstd/ZstdDecompressorBenchmark.java‎

Lines changed: 42 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,16 @@ public class ZstdDecompressorBenchmark {
2828
private static final int ZSTD_BUFFER_SIZE = DiscordZstdDecompressor.DEFAULT_BUFFER_SIZE;
2929
private static final int ZLIB_BUFFER_SIZE = 2048; // JDA default
3030

31+
@State(Scope.Benchmark)
32+
public static class ShardsState {
33+
public List<List<TestChunks.Chunk>> shards;
34+
35+
@Setup
36+
public void setup() {
37+
shards = TestChunks.get();
38+
}
39+
}
40+
3141
@State(Scope.Benchmark)
3242
public static class ZstdDecompressorState {
3343
@Param({"jni"})
@@ -50,34 +60,28 @@ public void tearDown() {
5060
}
5161
}
5262

53-
@State(Scope.Benchmark)
54-
public static class ZstdChunksState {
55-
public List<TestChunks.Chunk> chunks;
56-
57-
@Setup
58-
public void setup() {
59-
chunks = TestChunks.get(TestChunks.Compression.ZSTD);
60-
}
61-
}
62-
6363
@Benchmark
64-
public void zstd(ZstdDecompressorState decompressorState, ZstdChunksState chunksState, Blackhole blackhole) {
64+
public void zstd(ZstdDecompressorState decompressorState, ShardsState shardsState, Blackhole blackhole) {
6565
var decompressor = decompressorState.decompressor;
66-
decompressor.reset();
67-
// Can't make a benchmark per-message (so we can see scaling based on message sizes
68-
// as this uses a streaming decompressor, meaning this requires previous inputs
69-
for (TestChunks.Chunk chunk : chunksState.chunks)
70-
blackhole.consume(DataObject.fromJson(decompressor.decompress(chunk.getCompressed())));
66+
for (List<TestChunks.Chunk> shard : shardsState.shards) {
67+
decompressor.reset();
68+
// Can't make a benchmark per-message (so we can see scaling based on message sizes
69+
// as this uses a streaming decompressor, meaning this requires previous inputs
70+
for (TestChunks.Chunk chunk : shard)
71+
blackhole.consume(DataObject.fromJson(decompressor.decompress(chunk.zstdCompressed())));
72+
}
7173
}
7274

7375
@Benchmark
74-
public void zstdNoDeser(ZstdDecompressorState decompressorState, ZstdChunksState chunksState, Blackhole blackhole) {
76+
public void zstdNoDeser(ZstdDecompressorState decompressorState, ShardsState shardsState, Blackhole blackhole) {
7577
var decompressor = decompressorState.decompressor;
76-
decompressor.reset();
77-
// Can't make a benchmark per-message (so we can see scaling based on message sizes
78-
// as this uses a streaming decompressor, meaning this requires previous inputs
79-
for (TestChunks.Chunk chunk : chunksState.chunks)
80-
blackhole.consume(decompressor.decompress(chunk.getCompressed()));
78+
for (List<TestChunks.Chunk> shard : shardsState.shards) {
79+
decompressor.reset();
80+
// Can't make a benchmark per-message (so we can see scaling based on message sizes
81+
// as this uses a streaming decompressor, meaning this requires previous inputs
82+
for (TestChunks.Chunk chunk : shard)
83+
blackhole.consume(decompressor.decompress(chunk.zstdCompressed()));
84+
}
8185
}
8286

8387

@@ -92,35 +96,29 @@ public void setup() {
9296
}
9397
}
9498

95-
@State(Scope.Benchmark)
96-
public static class ZlibChunksState {
97-
public List<TestChunks.Chunk> chunks;
98-
99-
@Setup
100-
public void setup() {
101-
chunks = TestChunks.get(TestChunks.Compression.ZLIB);
102-
}
103-
}
104-
10599
@Benchmark
106-
public void zlib(ZlibDecompressorState decompressorState, ZlibChunksState chunksState, Blackhole blackhole) throws DataFormatException {
100+
public void zlib(ZlibDecompressorState decompressorState, ShardsState shardsState, Blackhole blackhole) throws DataFormatException {
107101
var decompressor = decompressorState.decompressor;
108-
decompressor.reset();
109-
// Can't make a benchmark per-message (so we can see scaling based on message sizes
110-
// as this uses a streaming decompressor, meaning this requires previous inputs
111-
for (TestChunks.Chunk chunk : chunksState.chunks) {
112-
blackhole.consume(DataObject.fromJson(decompressor.decompress(chunk.getCompressed())));
102+
for (List<TestChunks.Chunk> shard : shardsState.shards) {
103+
decompressor.reset();
104+
// Can't make a benchmark per-message (so we can see scaling based on message sizes
105+
// as this uses a streaming decompressor, meaning this requires previous inputs
106+
for (TestChunks.Chunk chunk : shard) {
107+
blackhole.consume(DataObject.fromJson(decompressor.decompress(chunk.zlibCompressed())));
108+
}
113109
}
114110
}
115111

116112
@Benchmark
117-
public void zlibNoDeser(ZlibDecompressorState decompressorState, ZlibChunksState chunksState, Blackhole blackhole) throws DataFormatException {
113+
public void zlibNoDeser(ZlibDecompressorState decompressorState, ShardsState shardsState, Blackhole blackhole) throws DataFormatException {
118114
var decompressor = decompressorState.decompressor;
119-
decompressor.reset();
120-
// Can't make a benchmark per-message (so we can see scaling based on message sizes
121-
// as this uses a streaming decompressor, meaning this requires previous inputs
122-
for (TestChunks.Chunk chunk : chunksState.chunks) {
123-
blackhole.consume(decompressor.decompress(chunk.getCompressed()));
115+
for (List<TestChunks.Chunk> shard : shardsState.shards) {
116+
decompressor.reset();
117+
// Can't make a benchmark per-message (so we can see scaling based on message sizes
118+
// as this uses a streaming decompressor, meaning this requires previous inputs
119+
for (TestChunks.Chunk chunk : shard) {
120+
blackhole.consume(decompressor.decompress(chunk.zlibCompressed()));
121+
}
124122
}
125123
}
126124

‎benchmarks/src/jmh/java/dev/freya02/discord/zstd/ZstdStreamingBenchmark.java‎

Lines changed: 52 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,16 @@
2424
@Fork(1)
2525
public class ZstdStreamingBenchmark {
2626

27+
@State(Scope.Benchmark)
28+
public static class ShardsState {
29+
public List<List<TestChunks.Chunk>> shards;
30+
31+
@Setup
32+
public void setup() {
33+
shards = TestChunks.get();
34+
}
35+
}
36+
2737
@State(Scope.Benchmark)
2838
public static class ZstdDecompressorState {
2939
@Param({"jni"})
@@ -48,38 +58,32 @@ public void tearDown() {
4858
}
4959
}
5060

51-
@State(Scope.Benchmark)
52-
public static class ZstdChunksState {
53-
public List<TestChunks.Chunk> chunks;
54-
55-
@Setup
56-
public void setup() {
57-
chunks = TestChunks.get(TestChunks.Compression.ZSTD);
58-
}
59-
}
60-
6161
@Benchmark
62-
public void zstd(ZstdDecompressorState decompressorState, ZstdChunksState chunksState, Blackhole blackhole) throws IOException {
62+
public void zstd(ZstdDecompressorState decompressorState, ShardsState shardsState, Blackhole blackhole) throws IOException {
6363
var context = decompressorState.context;
64-
context.reset();
65-
for (TestChunks.Chunk chunk : chunksState.chunks) {
66-
try (InputStream inputStream = context.createInputStream(chunk.getCompressed())) {
67-
blackhole.consume(DataObject.fromJson(inputStream));
64+
for (List<TestChunks.Chunk> shard : shardsState.shards) {
65+
context.reset();
66+
for (TestChunks.Chunk chunk : shard) {
67+
try (InputStream inputStream = context.createInputStream(chunk.zstdCompressed())) {
68+
blackhole.consume(DataObject.fromJson(inputStream));
69+
}
6870
}
6971
}
7072
}
7173

7274
@Benchmark
73-
public void zstdNoDeser(ZstdDecompressorState decompressorState, ZstdChunksState chunksState, Blackhole blackhole) throws IOException {
75+
public void zstdNoDeser(ZstdDecompressorState decompressorState, ShardsState shardsState, Blackhole blackhole) throws IOException {
7476
var context = decompressorState.context;
75-
context.reset();
76-
for (TestChunks.Chunk chunk : chunksState.chunks) {
77-
try (InputStream inputStream = context.createInputStream(chunk.getCompressed())) {
78-
while (true) {
79-
var read = inputStream.read(decompressorState.buf);
80-
blackhole.consume(decompressorState.buf);
81-
if (read <= 0) {
82-
break;
77+
for (List<TestChunks.Chunk> shard : shardsState.shards) {
78+
context.reset();
79+
for (TestChunks.Chunk chunk : shard) {
80+
try (InputStream inputStream = context.createInputStream(chunk.zstdCompressed())) {
81+
while (true) {
82+
var read = inputStream.read(decompressorState.buf);
83+
blackhole.consume(decompressorState.buf);
84+
if (read <= 0) {
85+
break;
86+
}
8387
}
8488
}
8589
}
@@ -101,49 +105,39 @@ public void setup() {
101105
}
102106
}
103107

104-
@State(Scope.Benchmark)
105-
public static class ZlibChunksState {
106-
public List<TestChunks.Chunk> chunks;
107-
108-
@Setup
109-
public void setup() {
110-
chunks = TestChunks.get(TestChunks.Compression.ZLIB);
111-
}
112-
}
113-
114108
@Benchmark
115-
public void zlib(ZlibDecompressorState decompressorState, ZlibChunksState chunksState, Blackhole blackhole) throws IOException {
109+
public void zlib(ZlibDecompressorState decompressorState, ShardsState shardsState, Blackhole blackhole) throws IOException {
116110
var decompressor = decompressorState.decompressor;
117-
decompressor.reset();
118-
// Can't make a benchmark per-message (so we can see scaling based on message sizes
119-
// as this uses a streaming decompressor, meaning this requires previous inputs
120-
for (TestChunks.Chunk chunk : chunksState.chunks) {
121-
try (InputStream inputStream = decompressor.createInputStream(chunk.getCompressed())) {
122-
blackhole.consume(DataObject.fromJson(inputStream));
111+
for (List<TestChunks.Chunk> shard : shardsState.shards) {
112+
decompressor.reset();
113+
for (TestChunks.Chunk chunk : shard) {
114+
try (InputStream inputStream = decompressor.createInputStream(chunk.zlibCompressed())) {
115+
blackhole.consume(DataObject.fromJson(inputStream));
116+
}
123117
}
124118
}
125119
}
126120

127121
@Benchmark
128-
public void zlibNoDeser(ZlibDecompressorState decompressorState, ZlibChunksState chunksState, Blackhole blackhole) throws IOException {
122+
public void zlibNoDeser(ZlibDecompressorState decompressorState, ShardsState shardsState, Blackhole blackhole) throws IOException {
129123
var decompressor = decompressorState.decompressor;
130-
decompressor.reset();
131124
var bytes = decompressorState.buf;
132-
// Can't make a benchmark per-message (so we can see scaling based on message sizes
133-
// as this uses a streaming decompressor, meaning this requires previous inputs
134-
for (TestChunks.Chunk chunk : chunksState.chunks) {
135-
int currentlyDecompressedSize = 0;
136-
int expectedDecompressedSize = chunk.getDecompressed().length;
137-
try (InputStream inputStream = decompressor.createInputStream(chunk.getCompressed())) {
138-
// This is pretty stupid, #available() returns 1 even when there is no output to be read,
139-
// we want to avoid handling EOFException as it may be slow and does not represent real world usage,
140-
// checking `read < buf.length` is not viable since it can store data internally and returned in the next call.
141-
// So, we instead decompress until we have the known decompressed data length.
142-
do {
143-
var read = inputStream.read(bytes);
144-
currentlyDecompressedSize += read;
145-
blackhole.consume(bytes);
146-
} while (currentlyDecompressedSize < expectedDecompressedSize);
125+
for (List<TestChunks.Chunk> shard : shardsState.shards) {
126+
decompressor.reset();
127+
for (TestChunks.Chunk chunk : shard) {
128+
int currentlyDecompressedSize = 0;
129+
int expectedDecompressedSize = chunk.decompressed().length;
130+
try (InputStream inputStream = decompressor.createInputStream(chunk.zlibCompressed())) {
131+
// This is pretty stupid, #available() returns 1 even when there is no output to be read,
132+
// we want to avoid handling EOFException as it may be slow and does not represent real world usage,
133+
// checking `read < buf.length` is not viable since it can store data internally and returned in the next call.
134+
// So, we instead decompress until we have the known decompressed data length.
135+
do {
136+
var read = inputStream.read(bytes);
137+
currentlyDecompressedSize += read;
138+
blackhole.consume(bytes);
139+
} while (currentlyDecompressedSize < expectedDecompressedSize);
140+
}
147141
}
148142
}
149143
}

‎buildSrc/build.gradle.kts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,4 +11,11 @@ repositories {
1111
dependencies {
1212
implementation(gradleApi())
1313
implementation(libs.jreleaser)
14+
implementation(libs.zstd.jni)
15+
}
16+
17+
kotlin {
18+
compilerOptions {
19+
freeCompilerArgs.add("-Xcontext-parameters")
20+
}
1421
}

0 commit comments

Comments
 (0)