Merge remote-tracking branch 'origin/main' into doris_main

Author: rschu1ze

README.md

@@ -24,7 +24,7 @@ You can easily reproduce every test (although for some systems it may take from
 
 Best efforts should be taken to understand the details of every tested system for a fair comparison. It is allowed to apply various [indexing methods](https://clickhouse.com/blog/json-bench-clickhouse-vs-mongodb-elasticsearch-duckdb-postgresql#some-json-paths-can-be-used-for-indexes-and-data-sorting) whenever appropriate.
 
-It is [not allowed](https://clickhouse.com/blog/json-bench-clickhouse-vs-mongodb-elasticsearch-duckdb-postgresql#no-query-results-cache) to use query results caching or flatten JSON into multiple non-JSON colums at insertion type.
+It is [not allowed](https://clickhouse.com/blog/json-bench-clickhouse-vs-mongodb-elasticsearch-duckdb-postgresql#no-query-results-cache) to use query results caching or flatten JSON into multiple non-JSON colums at insertion time.
 
 Some databases do have a JSON data type but they flatten nested JSON documents at insertion time to a single level (typically using `.` as separator between levels). We consider this a grey zone. On the one hand, this removes the possibility to restore the original documents, on the other hand, flattening may in many practical situations be acceptable. The dashboard allows to filter out databases which do not retain the document structure (i.e. which flatten).
 

generate-results.sh

@@ -4,11 +4,15 @@
 # Note: editing HTML with sed may look strange, but at least we avoid using node.js and npm, and that's good.
 
 # This is needed on Mac OS. Do `brew install coreutils`.
-[ -n "$HOMEBREW_PREFIX" ] && PATH="${HOMEBREW_PREFIX}/opt/coreutils/libexec/gnubin:${PATH}"
-if command -v gsed >/dev/null 2>&1
-then
+if [[ "$(uname)" == "Darwin" ]]; then
+  if ! command -v gsed >/dev/null 2>&1
+  then
+    echo "On macOS, please install GNU sed through homebrew."
+    exit 1
+  else
     shopt -s expand_aliases
     alias sed='gsed'
+  fi
 fi
 
 (

index.html

@@ -369,14 +369,10 @@
 ,{"system":"SingleStore","version":"5.7.32","os":"Ubuntu 24.04","date":"2025-03-14","machine":"m6i.8xlarge, 10000gib gp3","retains_structure":"yes","tags":[],"dataset_size":100000000,"num_loaded_documents":91000000,"total_size":25757172429,"data_size":25757172429,"index_size":0,"result":[[6.471,5.066,5.606],[37.551,25.246,25.060],[14.634,12.810,12.786],[34.688,3.691,3.702],[34.735,4.934,4.713]],"memory_usage":[[null,null,null],[null,null,null],[null,null,null],[null,null,null],[null,null,null]],"source":"singlestore/results/m6i.8xlarge_bluesky_100m.json"}
 ,{"system":"SingleStore","version":"5.7.32","os":"Ubuntu 24.04","date":"2025-03-14","machine":"m6i.8xlarge, 10000gib gp3","retains_structure":"yes","tags":[],"dataset_size":10000000,"num_loaded_documents":7000000,"total_size":1943790266,"data_size":1943790266,"index_size":0,"result":[[0.658,0.444,0.494],[2.831,2.647,2.836],[1.437,1.456,1.124],[1.253,0.412,0.434],[1.347,0.553,0.587]],"memory_usage":[[null,null,null],[null,null,null],[null,null,null],[null,null,null],[null,null,null]],"source":"singlestore/results/m6i.8xlarge_bluesky_10m.json"}
 ,{"system":"SingleStore","version":"5.7.32","os":"Ubuntu 24.04","date":"2025-03-14","machine":"m6i.8xlarge, 10000gib gp3","retains_structure":"yes","tags":[],"dataset_size":1000000,"num_loaded_documents":1000000,"total_size":275466582,"data_size":275466582,"index_size":0,"result":[[0.206,0.083,0.098],[0.615,0.544,0.452],[0.366,0.268,0.149],[0.259,0.111,0.106],[0.324,0.167,0.171]],"memory_usage":[[null,null,null],[null,null,null],[null,null,null],[null,null,null],[null,null,null]],"source":"singlestore/results/m6i.8xlarge_bluesky_1m.json"}
-,{"system":"Starrocks (default)","version":"3.4.0-e94580b","os":"Ubuntu 24.04","date":"2025-03-24","machine":"m6i.8xlarge, 10000gib gp3","retains_structure":"yes","tags":[],"dataset_size":1000000000,"num_loaded_documents":804000000,"total_size":158929000000,"result":[[2.27,1.24,1.21],[17.81,10.67,10.20],[7.38,6.78,7.62],[7.24,null,null],[null,null,null]],"source":"starrocks/results/m6i.8xlarge_bluesky_1000m_default.json"}
-,{"system":"Starrocks (materialized)","version":"3.4.0-e94580b","os":"Ubuntu 24.04","date":"2025-03-24","machine":"m6i.8xlarge, 10000gib gp3","retains_structure":"yes","tags":[],"dataset_size":1000000000,"num_loaded_documents":997000000,"total_size":191541000000,"result":[[1.75,1.56,1.54],[49.75,41.61,31.38],[12.90,12.58,5.76],[5.64,6.21,6.03],[null,null,null]],"source":"starrocks/results/m6i.8xlarge_bluesky_1000m_materialized.json"}
-,{"system":"Starrocks (default)","version":"3.4.0-e94580b","os":"Ubuntu 24.04","date":"2025-03-24","machine":"m6i.8xlarge, 10000gib gp3","retains_structure":"yes","tags":[],"dataset_size":100000000,"num_loaded_documents":91000000,"total_size":17109000000,"result":[[0.61,0.16,0.16],[19.26,7.12,7.18],[1.12,1.08,1.08],[0.55,0.55,0.54],[0.60,0.60,0.60]],"source":"starrocks/results/m6i.8xlarge_bluesky_100m_default.json"}
-,{"system":"Starrocks (materialized)","version":"3.4.0-e94580b","os":"Ubuntu 24.04","date":"2025-03-24","machine":"m6i.8xlarge, 10000gib gp3","retains_structure":"yes","tags":[],"dataset_size":100000000,"num_loaded_documents":100000000,"total_size":16190000000,"result":[[0.21,0.17,0.18],[8.38,2.19,2.17],[2.16,1.10,1.06],[6.62,0.43,0.45],[0.48,0.48,0.49]],"source":"starrocks/results/m6i.8xlarge_bluesky_100m_materialized.json"}
-,{"system":"Starrocks (default)","version":"3.4.0-e94580b","os":"Ubuntu 24.04","date":"2025-03-24","machine":"m6i.8xlarge, 10000gib gp3","retains_structure":"yes","tags":[],"dataset_size":10000000,"num_loaded_documents":7000000,"total_size":824028000,"result":[[0.03,0.02,0.03],[0.52,0.50,0.48],[0.25,0.25,0.18],[0.11,0.11,0.10],[0.11,0.12,0.12]],"source":"starrocks/results/m6i.8xlarge_bluesky_10m_default.json"}
-,{"system":"Starrocks (materialized)","version":"3.4.0-e94580b","os":"Ubuntu 24.04","date":"2025-03-24","machine":"m6i.8xlarge, 10000gib gp3","retains_structure":"yes","tags":[],"dataset_size":10000000,"num_loaded_documents":10000000,"total_size":616175000,"result":[[0.09,0.13,0.12],[0.34,0.33,0.33],[0.22,0.20,0.26],[0.11,0.10,0.10],[0.11,0.10,0.11]],"source":"starrocks/results/m6i.8xlarge_bluesky_10m_materialized.json"}
-,{"system":"Starrocks (default)","version":"3.4.0-e94580b","os":"Ubuntu 24.04","date":"2025-03-24","machine":"m6i.8xlarge, 10000gib gp3","retains_structure":"yes","tags":[],"dataset_size":1000000,"num_loaded_documents":1000000,"total_size":1,"result":[[0.05,0.05,0.06],[0.19,0.08,0.07],[0.13,0.13,0.13],[0.07,0.07,0.07],[0.07,0.07,0.07]],"source":"starrocks/results/m6i.8xlarge_bluesky_1m_default.json"}
-,{"system":"Starrocks (materialized)","version":"3.4.0-e94580b","os":"Ubuntu 24.04","date":"2025-03-24","machine":"m6i.8xlarge, 10000gib gp3","retains_structure":"yes","tags":[],"dataset_size":1000000,"num_loaded_documents":1000000,"total_size":1,"result":[[0.06,0.05,0.04],[0.13,0.06,0.08],[0.14,0.13,0.13],[0.12,0.05,0.05],[0.05,0.05,0.05]],"source":"starrocks/results/m6i.8xlarge_bluesky_1m_materialized.json"}
+,{"system":"Starrocks","version":"3.4.0-e94580b","os":"Ubuntu 24.04","date":"2025-03-24","machine":"m6i.8xlarge, 10000gib gp3","retains_structure":"yes","tags":[],"dataset_size":1000000000,"num_loaded_documents":804000000,"total_size":158929000000,"result":[[2.27,1.24,1.21],[17.81,10.67,10.20],[7.38,6.78,7.62],[7.24,null,null],[null,null,null]],"source":"starrocks/results/m6i.8xlarge_bluesky_1000m.json"}
+,{"system":"Starrocks","version":"3.4.0-e94580b","os":"Ubuntu 24.04","date":"2025-03-24","machine":"m6i.8xlarge, 10000gib gp3","retains_structure":"yes","tags":[],"dataset_size":100000000,"num_loaded_documents":91000000,"total_size":17109000000,"result":[[0.61,0.16,0.16],[19.26,7.12,7.18],[1.12,1.08,1.08],[0.55,0.55,0.54],[0.60,0.60,0.60]],"source":"starrocks/results/m6i.8xlarge_bluesky_100m.json"}
+,{"system":"Starrocks","version":"3.4.0-e94580b","os":"Ubuntu 24.04","date":"2025-03-24","machine":"m6i.8xlarge, 10000gib gp3","retains_structure":"yes","tags":[],"dataset_size":10000000,"num_loaded_documents":7000000,"total_size":824028000,"result":[[0.03,0.02,0.03],[0.52,0.50,0.48],[0.25,0.25,0.18],[0.11,0.11,0.10],[0.11,0.12,0.12]],"source":"starrocks/results/m6i.8xlarge_bluesky_10m.json"}
+,{"system":"Starrocks","version":"3.4.0-e94580b","os":"Ubuntu 24.04","date":"2025-03-24","machine":"m6i.8xlarge, 10000gib gp3","retains_structure":"yes","tags":[],"dataset_size":1000000,"num_loaded_documents":1000000,"total_size":1,"result":[[0.05,0.05,0.06],[0.19,0.08,0.07],[0.13,0.13,0.13],[0.07,0.07,0.07],[0.07,0.07,0.07]],"source":"starrocks/results/m6i.8xlarge_bluesky_1m.json"}
 ,{"system":"VictoriaLogs","version":"v1.17.0-victorialogs","os":"Ubuntu 24.04","date":"2025-03-28","machine":"m6i.8xlarge, 10000gib gp3","retains_structure":"no","tags":[],"dataset_size":1000000000,"num_loaded_documents":999999241,"total_size":121860870843,"data_size":108388327979,"index_size":13472542864,"result":[[1.076,0.029,0.028],[98.958,20.591,19.794],[14.423,1.607,1.609],[5.065,5.185,5.216],[7.22,6.963,7.351]],"source":"victorialogs/results/m6i.8xlarge_bluesky_1000m.json"}
 ,{"system":"VictoriaLogs","version":"v1.17.0-victorialogs","os":"Ubuntu 24.04","date":"2025-03-28","machine":"m6i.8xlarge, 10000gib gp3","retains_structure":"no","tags":[],"dataset_size":100000000,"num_loaded_documents":99999968,"total_size":11984344799,"data_size":10684518543,"index_size":1299826256,"result":[[0.097,0.009,0.009],[9.057,2.721,2.406],[0.474,0.151,0.148],[0.542,0.464,0.453],[0.735,0.623,0.664]],"source":"victorialogs/results/m6i.8xlarge_bluesky_100m.json"}
 ,{"system":"VictoriaLogs","version":"v1.17.0-victorialogs","os":"Ubuntu 24.04","date":"2025-03-28","machine":"m6i.8xlarge, 10000gib gp3","retains_structure":"no","tags":[],"dataset_size":10000000,"num_loaded_documents":9999994,"total_size":1242217952,"data_size":1108712800,"index_size":133505152,"result":[[0.086,0.007,0.007],[0.47,0.241,0.329],[0.095,0.027,0.028],[0.199,0.142,0.128],[0.211,0.148,0.176]],"source":"victorialogs/results/m6i.8xlarge_bluesky_10m.json"}

starrocks/create_and_load.sh

@@ -3,22 +3,20 @@
 # If you change something in this file, please change also in doris/create_and_load.sh.
 
 # Check if the required arguments are provided
-if [[ $# -lt 7 ]]; then
-    echo "Usage: $0 <DB_NAME> <TABLE_NAME> <DDL_FILE> <DATA_DIRECTORY> <NUM_FILES> <SUCCESS_LOG> <ERROR_LOG>"
+if [[ $# -lt 6 ]]; then
+    echo "Usage: $0 <DB_NAME> <TABLE_NAME> <DATA_DIRECTORY> <NUM_FILES> <SUCCESS_LOG> <ERROR_LOG>"
     exit 1
 fi
 
 # Arguments
 DB_NAME="$1"
 TABLE_NAME="$2"
-DDL_FILE="$3"
-DATA_DIRECTORY="$4"
-NUM_FILES="$5"
-SUCCESS_LOG="$6"
-ERROR_LOG="$7"
+DATA_DIRECTORY="$3"
+NUM_FILES="$4"
+SUCCESS_LOG="$5"
+ERROR_LOG="$6"
 
 # Validate arguments
-[[ ! -f "$DDL_FILE" ]] && { echo "Error: DDL file '$DDL_FILE' does not exist."; exit 1; }
 [[ ! -d "$DATA_DIRECTORY" ]] && { echo "Error: Data directory '$DATA_DIRECTORY' does not exist."; exit 1; }
 [[ ! "$NUM_FILES" =~ ^[0-9]+$ ]] && { echo "Error: NUM_FILES must be a positive integer."; exit 1; }
 
@@ -27,7 +25,7 @@ echo "Create database"
 mysql -P 9030 -h 127.0.0.1 -u root -e "CREATE DATABASE IF NOT EXISTS $DB_NAME"
 
 echo "Execute DDL"
-mysql -P 9030 -h 127.0.0.1 -u root $DB_NAME < "$DDL_FILE"
+mysql -P 9030 -h 127.0.0.1 -u root $DB_NAME < "ddl.sql"
 
 echo "Load data"
 ./load_data.sh "$DATA_DIRECTORY" "$DB_NAME" "$TABLE_NAME" "$NUM_FILES" "$SUCCESS_LOG" "$ERROR_LOG"

starrocks/ddl_default.sql renamed to starrocks/ddl.sql

@@ -38,47 +38,38 @@ fi;
 
 benchmark() {
     local size=$1
-    local suffix=$2
     # Check DATA_DIRECTORY contains the required number of files to run the benchmark
     file_count=$(find "$DATA_DIRECTORY" -type f | wc -l)
     if (( file_count < size )); then
         echo "Error: Not enough files in '$DATA_DIRECTORY'. Required: $size, Found: $file_count."
         exit 1
     fi
-    ./create_and_load.sh "bluesky_${size}m_${suffix}" bluesky "ddl_${suffix}.sql" "$DATA_DIRECTORY" "$size" "$SUCCESS_LOG" "$ERROR_LOG"
-    ./total_size.sh "bluesky_${size}m_${suffix}" bluesky | tee "${OUTPUT_PREFIX}_bluesky_${size}m_${suffix}.total_size"
-    ./count.sh "bluesky_${size}m_${suffix}" bluesky | tee "${OUTPUT_PREFIX}_bluesky_${size}m_${suffix}.count"
-    ./physical_query_plans.sh "bluesky_${size}m_${suffix}" | tee "${OUTPUT_PREFIX}_bluesky_${size}m_${suffix}.physical_query_plans"
-    ./benchmark.sh "bluesky_${size}m_${suffix}" "${OUTPUT_PREFIX}_bluesky_${size}m_${suffix}.results_runtime" "${OUTPUT_PREFIX}_bluesky_${size}m_${suffix}.results_memory_usage"
-    ./drop_table.sh "bluesky_${size}m_${suffix}" bluesky
+    ./create_and_load.sh "bluesky_${size}m" bluesky "$DATA_DIRECTORY" "$size" "$SUCCESS_LOG" "$ERROR_LOG"
+    ./total_size.sh "bluesky_${size}m" bluesky | tee "${OUTPUT_PREFIX}_bluesky_${size}m.total_size"
+    ./count.sh "bluesky_${size}m" bluesky | tee "${OUTPUT_PREFIX}_bluesky_${size}m.count"
+    ./physical_query_plans.sh "bluesky_${size}m" | tee "${OUTPUT_PREFIX}_bluesky_${size}m.physical_query_plans"
+    ./benchmark.sh "bluesky_${size}m" "${OUTPUT_PREFIX}_bluesky_${size}m.results_runtime" "${OUTPUT_PREFIX}_bluesky_${size}m.results_memory_usage"
+    ./drop_table.sh "bluesky_${size}m" bluesky
 }
 
 case $CHOICE in
     2)
-        benchmark 10 default
-        benchmark 10 materialized
+        benchmark 10
         ;;
     3)
-        benchmark 100 default
-        benchmark 100 materialized
+        benchmark 100
         ;;
     4)
-        benchmark 1000 default
-        benchmark 1000 materialized
+        benchmark 1000
         ;;
     5)
-        benchmark 1 materialized
-        benchmark 1 default
-        benchmark 10 materialized
-        benchmark 10 default
-        benchmark 100 materialized
-        benchmark 100 default
-        benchmark 1000 materialized
-        benchmark 1000 default
+        benchmark 1
+        benchmark 10
+        benchmark 100
+        benchmark 1000
         ;;
     *)
-        benchmark 1 materialized
-        benchmark 1 default
+        benchmark 1
         ;;
 esac
 

starrocks/ddl_materialized.sql

@@ -1,5 +1,5 @@
 {
-  "system": "Starrocks (default)",
+  "system": "Starrocks",
   "version": "3.4.0-e94580b",
   "os": "Ubuntu 24.04",
   "date": "2025-03-24",

starrocks/main.sh

@@ -1,5 +1,5 @@
 {
-  "system": "Starrocks (default)",
+  "system": "Starrocks",
   "version": "3.4.0-e94580b",
   "os": "Ubuntu 24.04",
   "date": "2025-03-24",