make each input file 8mb

Author: parkertimmins

benchmarks/src/main/java/org/elasticsearch/benchmark/common/compress/FSSTCompressBenchmark.java

@@ -47,26 +47,30 @@ public static class CompressionMetrics {
         public double compressionRatio;
     }
 
+    private static final int MB_8 = 8 * 1024 * 1024;
+    private byte[] concatenateTo8mb(byte[] contentBytes) {
+        byte[] bytes = new byte[MB_8 + 8];
+        int i = 0;
+        while (i < MB_8) {
+            int remaining = MB_8 - i;
+            int len = Math.min(contentBytes.length, remaining);
+            System.arraycopy(contentBytes, 0, bytes, i, len);
+            i += len;
+        }
+        return bytes;
+    }
+
     @Setup(Level.Trial)
     public void setup() throws IOException {
         String content = Files.readString(Path.of(dataset), StandardCharsets.UTF_8);
-        byte[] bytes = FSST.toBytes(content);
-        byte[] bytes2 = new byte[bytes.length + 8];
-        System.arraycopy(bytes, 0, bytes2, 0, bytes.length);
-        input = bytes2;
-        offsets = new int[] { 0, bytes.length };
 
-        outBuf = new byte[bytes.length];
+        byte[] contentBytes = FSST.toBytes(content);
+        input = concatenateTo8mb(contentBytes);
+        offsets = new int[]{0, MB_8};
+        outBuf = new byte[MB_8];
         outOffsets = new int[2];
     }
 
-    @Benchmark
-    public void makeSample(Blackhole bh, CompressionMetrics metrics) {
-        List<byte[]> sample = FSST.makeSample(input, offsets);
-        var symbolTable = FSST.SymbolTable.buildSymbolTable(sample);
-        bh.consume(sample);
-    }
-
     @Benchmark
     public void compressFSST(Blackhole bh, CompressionMetrics metrics) {
         List<byte[]> sample = FSST.makeSample(input, offsets);
@@ -96,19 +100,19 @@ public void compressLZ4Fast(Blackhole bh, CompressionMetrics metrics) throws IOE
         metrics.compressionRatio = compressedSize / (double) inputSize;
     }
 
-    @Benchmark
-    public void compressLZ4High(Blackhole bh, CompressionMetrics metrics) throws IOException {
-        int inputSize = offsets[1];
-
-        var dataInput = new ByteBuffersDataInput(List.of(ByteBuffer.wrap(input)));
-        var dataOutput = new ByteArrayDataOutput(outBuf);
-
-        Compressor compressor = CompressionMode.HIGH_COMPRESSION.newCompressor();
-        compressor.compress(dataInput, dataOutput);
-
-        long compressedSize = dataOutput.getPosition();
-        bh.consume(dataOutput);
-
-        metrics.compressionRatio = compressedSize / (double) inputSize;
-    }
+//    @Benchmark
+//    public void compressLZ4High(Blackhole bh, CompressionMetrics metrics) throws IOException {
+//        int inputSize = offsets[1];
+//
+//        var dataInput = new ByteBuffersDataInput(List.of(ByteBuffer.wrap(input)));
+//        var dataOutput = new ByteArrayDataOutput(outBuf);
+//
+//        Compressor compressor = CompressionMode.HIGH_COMPRESSION.newCompressor();
+//        compressor.compress(dataInput, dataOutput);
+//
+//        long compressedSize = dataOutput.getPosition();
+//        bh.consume(dataOutput);
+//
+//        metrics.compressionRatio = compressedSize / (double) inputSize;
+//    }
 }

benchmarks/src/main/java/org/elasticsearch/benchmark/common/compress/FSSTDecompressBenchmark.java

@@ -25,6 +25,7 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 
@@ -36,7 +37,8 @@
 @State(Scope.Benchmark)
 public class FSSTDecompressBenchmark {
 
-    @Param({ "fsst", "lz4_high", "lz4_fast" })
+//    @Param({ "fsst", "lz4_high", "lz4_fast" })
+    @Param({"fsst", "lz4_fast"})
     public String compressionType;
 
     @Param("")
@@ -58,14 +60,26 @@ public class FSSTDecompressBenchmark {
     // fsst specific
     private FSST.SymbolTable symbolTable;
 
+    private static final int MB_8 = 8 * 1024 * 1024;
+    private byte[] concatenateTo8mb(byte[] contentBytes) {
+        byte[] bytes = new byte[MB_8 + 8];
+        int i = 0;
+        while (i < MB_8) {
+            int remaining = MB_8 - i;
+            int len = Math.min(contentBytes.length, remaining);
+            System.arraycopy(contentBytes, 0, bytes, i, len);
+            i += len;
+        }
+        return bytes;
+    }
+
     @Setup(Level.Trial)
     public void setup() throws IOException {
         String content = Files.readString(Path.of(dataset), StandardCharsets.UTF_8);
-        byte[] bytes = FSST.toBytes(content);
-        originalSize = bytes.length;
-        input = new byte[originalSize + 8];
-        offsets = new int[] { 0, bytes.length };
-        System.arraycopy(bytes, 0, input, 0, bytes.length);
+        byte[] contentBytes = FSST.toBytes(content);
+        originalSize = MB_8;
+        input = concatenateTo8mb(contentBytes);
+        offsets = new int[]{0, originalSize};
 
         outBuf = new byte[input.length];
         outOffsets = new int[2];