added dataset generation

Author: chernser

.github/workflows/benchmarks.yml

@@ -55,10 +55,12 @@ jobs:
           cache: "maven"
       - name: Build
         run: mvn --batch-mode --no-transfer-progress -Dj8 -DskipTests=true clean install
-      - name: Build
+      - name: Run Benchmarks
         run: | 
-          cd performance \
-          mvn compile exec:exec -Dexec.args="-classpath %classpath com.clickhouse.benchmark.BenchmarkRunner -m 3 -t 30 -b i"
+          cd performance;\
+          mvn compile exec:exec -Dexec.args="-classpath %classpath com.clickhouse.benchmark.data.DataSetGenerator \
+          -input sample_dataset.sql -name default -rows 100000"; \
+          mvn compile exec:exec -Dexec.args="-classpath %classpath com.clickhouse.benchmark.BenchmarkRunner -l 100000,10000 -m 1 -t 3 -b i -d file://default.csv"
       - name: Upload test results
         uses: actions/upload-artifact@v4
         if: success()

performance/README.md

@@ -8,6 +8,16 @@
 
 ### How to Run
 
+
+#### Generating Dataset
+
+```shell
+mvn compile exec:exec -Dexec.args="-classpath %classpath com.clickhouse.benchmark.data.DataSetGenerator \
+-input sample_dataset.sql -name default -rows 10"
+```
+
+#### Running Benchmarks 
+ 
 With default settings :
 ```shell
 mvn compile exec:exec
@@ -19,6 +29,8 @@ mvn compile exec:exec -Dexec.args="-classpath %classpath com.clickhouse.benchmar
 ```
 
 Other options:
+- "-d" - dataset name or file path (like `file://default.csv`)
+- "-l" - dataset limits to test coma separated (ex.: `-l 10000,10000`)
 - "-m" - number of measurement iterations
 - "-t" - time in seconds per iteration
 - "-b" - benchmark mask coma separated. Ex.: `-b writer,reader,i`. Default : `-b i,q`

performance/sample_dataset.sql

@@ -0,0 +1,40 @@
+CREATE TABLE big_columns_example
+(
+    -- Float64 columns
+    float_col_1 Float64,
+    float_col_2 Float64,
+    float_col_3 Float64,
+    float_col_4 Float64,
+    float_col_5 Float64,
+    float_col_6 Float64,
+    float_col_7 Float64,
+    float_col_8 Float64,
+    float_col_9 Float64,
+    float_col_10 Float64,
+
+    -- String columns
+    str_col_1 String,
+    str_col_2 String,
+    str_col_3 String,
+    str_col_4 String,
+    str_col_5 String,
+    str_col_6 String,
+    str_col_7 String,
+    str_col_8 String,
+    str_col_9 String,
+    str_col_10 String,
+
+    -- Int64 columns
+    int_col_1 Int64,
+    int_col_2 Int64,
+    int_col_3 Int64,
+    int_col_4 Int64,
+    int_col_5 Int64,
+    int_col_6 Int64,
+    int_col_7 Int64,
+    int_col_8 Int64,
+    int_col_9 Int64,
+    int_col_10 Int64
+)
+ENGINE = MergeTree
+ORDER BY tuple();

performance/src/main/java/com/clickhouse/benchmark/BenchmarkRunner.java

@@ -12,10 +12,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.HashMap;
-import java.util.Map;
-import java.util.SortedSet;
-import java.util.TreeSet;
+import java.util.*;
 import java.util.concurrent.TimeUnit;
 
 import static com.clickhouse.benchmark.TestEnvironment.isCloud;
@@ -35,15 +32,21 @@ public static void main(String[] args) throws Exception {
         final int measurementTime = Integer.parseInt(options.getOrDefault("-t", "" + (isCloud() ? 30 : 10)));
         final String resultFile = String.format("jmh-results-%s-%s.json", env, time);
         final String outputFile = String.format("jmh-results-%s-%s.out", env, time);
+        final String datasetName = options.getOrDefault("-d", "file://default.csv");
+        final String[] limits = options.getOrDefault("-l", "300000,100000,10000").split(",");
 
         System.out.println("Measurement iterations: " + measurementIterations);
         System.out.println("Measurement time: " + measurementTime + "s");
         System.out.println("Env: " + env);
+        System.out.println("Dataset: " + datasetName);
+        System.out.println("Limits: " + Arrays.asList(limits));
         System.out.println("Epoch Time: " + time);
 
         ChainedOptionsBuilder optBuilder = new OptionsBuilder()
                 .forks(1) // must be a fork. No fork only for debugging
                 .mode(Mode.SampleTime)
+                .param("datasetSourceName", datasetName)
+                .param("limit", limits)
                 .timeUnit(TimeUnit.MILLISECONDS)
                 .addProfiler(GCProfiler.class)
                 .addProfiler(MemPoolProfiler.class)
@@ -54,7 +57,8 @@ public static void main(String[] args) throws Exception {
                 .measurementTime(TimeValue.seconds(measurementTime))
                 .resultFormat(ResultFormatType.JSON)
                 .output(outputFile)
-                .result(resultFile);
+                .result(resultFile)
+                .shouldFailOnError(true);
 
         String testMask = options.getOrDefault("-b", "q,i");
         String[] testMaskParts = testMask.split(",");

performance/src/main/java/com/clickhouse/benchmark/data/DataSetGenerator.java

@@ -57,6 +57,7 @@ public static void main(String[] args) throws IOException {
                     if (i + 1 < args.length) {
                         datasetName = args[++i];
                     }
+                    break;
                 default:
                     // Ignore unknown flags
                     break;
@@ -93,7 +94,7 @@ public static void main(String[] args) throws IOException {
         LocalDateTime seriesStart = null;
         Duration seriesInterval = Duration.ofSeconds(1);
 
-        File outputFile = new File("dataset_" + System.currentTimeMillis() + ".csv");
+        File outputFile = new File(datasetName + ".csv");
         try (BufferedWriter writer = Files.newBufferedWriter(outputFile.toPath())) {
             writer.write(datasetName + "\n");
             writer.write("--Create Table Statement\n");
@@ -125,7 +126,7 @@ private static List<Column> parseCreateTable(String sql) throws Exception {
         int columnCount = 0;
         for (String line : reader.lines().collect(Collectors.toList())) {
             line = line.trim();
-            if (line.startsWith("CREATE TABLE") || line.startsWith("CREATE TEMPORARY TABLE")) {
+            if (line.startsWith("CREATE TABLE") || line.startsWith("CREATE TEMPORARY TABLE") || line.trim().startsWith("--")) {
                 continue; // Skip the CREATE TABLE line
             }
             if (line.startsWith(")")) {