update

Author: BiteTheDDDDt

doris/benchmark.sh

@@ -2,14 +2,14 @@
 
 # Check if the required arguments are provided
 if [[ $# -lt 3 ]]; then
-    echo "Usage: $0 <DB_NAME> <RESULT_FILE_RUNTIMES> <RESULT_FILE_MEMORY_USAGE>"
+    echo "Usage: $0 <DB_NAME> <RESULT_FILE_RUNTIMES> <QUERIES_FILE>"
     exit 1
 fi
 
 # Arguments
 DB_NAME="$1"
 RESULT_FILE_RUNTIMES="$2"
-RESULT_FILE_MEMORY_USAGE="$3"
+QUERIES_FILE="$3"
 
 # Construct the query log file name using $DB_NAME
 QUERY_LOG_FILE="query_log.txt"
@@ -18,7 +18,7 @@ QUERY_LOG_FILE="query_log.txt"
 echo "Running queries on database: $DB_NAME"
 
 # Run queries and log the output
-./run_queries.sh "$DB_NAME" 2>&1 | tee query_log.txt
+./run_queries.sh "$DB_NAME" "$QUERIES_FILE"  2>&1 | tee query_log.txt
 
 # Process the query log and prepare the result
 RESULT=$(cat query_log.txt | grep -oP 'Response time: \d+\.\d+ s' | sed -r -e 's/Response time: ([0-9]+\.[0-9]+) s/\1/' | \

doris/ddl_default.sql

@@ -0,0 +1,8 @@
+CREATE TABLE bluesky (
+    `id` BIGINT NOT NULL AUTO_INCREMENT,
+    `data` variant NOT NULL
+)
+DISTRIBUTED BY HASH(id) BUCKETS 32
+PROPERTIES (
+    "replication_num"="1"
+);

doris/ddl_generic.sql doris/ddl_materialized.sqldoris/ddl_generic.sql renamed to doris/ddl_materialized.sql

@@ -2,6 +2,3 @@
 wget --timestamping https://apache-doris-releases.oss-accelerate.aliyuncs.com/${DORIS_PACKAGE}.tar.gz
 mkdir ${DORIS_PACKAGE}
 tar -xvf ./${DORIS_PACKAGE}.tar.gz --strip-components 1 -C ./${DORIS_PACKAGE}
-
-echo "storage_page_cache_limit=60%" >> ./${DORIS_PACKAGE}/be/conf/be.conf
-echo "enable_java_support=false" >> ./${DORIS_PACKAGE}/be/conf/be.conf

doris/install.sh

@@ -48,29 +48,38 @@ benchmark() {
     ./create_and_load.sh "bluesky_${size}m_${suffix}" bluesky "ddl_${suffix}.sql" "$DATA_DIRECTORY" "$size" "$SUCCESS_LOG" "$ERROR_LOG"
     ./total_size.sh "bluesky_${size}m_${suffix}" bluesky | tee "${OUTPUT_PREFIX}_bluesky_${size}m_${suffix}.total_size"
     ./count.sh "bluesky_${size}m_${suffix}" bluesky | tee "${OUTPUT_PREFIX}_bluesky_${size}m_${suffix}.count"
-    ./benchmark.sh "bluesky_${size}m_${suffix}" "${OUTPUT_PREFIX}_bluesky_${size}m_${suffix}.results_runtime" "${OUTPUT_PREFIX}_bluesky_${size}m_${suffix}.results_memory_usage"
+    ./benchmark.sh "bluesky_${size}m_${suffix}" "${OUTPUT_PREFIX}_bluesky_${size}m_${suffix}.results_runtime" "queries_${suffix}.sql"
     ./drop_table.sh "bluesky_${size}m_${suffix}" bluesky
 }
 
 case $CHOICE in
     2)
-        benchmark 10 generic
+        benchmark 10 default
+        benchmark 10 materialized
         ;;
     3)
-        benchmark 100 generic
+        benchmark 100 default
+        benchmark 100 materialized
         ;;
     4)
-        benchmark 1000 generic
+        benchmark 1000 default
+        benchmark 1000 materialized
         ;;
     5)
-        benchmark 1 generic
-        benchmark 10 generic
-        benchmark 100 generic
-        benchmark 1000 generic
+        benchmark 1 materialized
+        benchmark 1 default
+        benchmark 10 materialized
+        benchmark 10 default
+        benchmark 100 materialized
+        benchmark 100 default
+        benchmark 1000 materialized
+        benchmark 1000 default
         ;;
     *)
-        benchmark 1 generic
+        benchmark 1 materialized
+        benchmark 1 default
         ;;
 esac
 
-./uninstall.sh
+./stop.sh
+#./uninstall.sh

doris/main.sh

@@ -0,0 +1,5 @@
+SELECT get_json_string(data, 'commit.collection') AS event, COUNT(*) AS count FROM bluesky GROUP BY event ORDER BY count DESC;
+SELECT get_json_string(data, 'commit.collection') AS event, COUNT(*) AS count, COUNT(DISTINCT get_json_string(data, 'did')) AS users FROM bluesky WHERE get_json_string(data, 'kind') = 'commit' AND get_json_string(data, 'commit.operation') = 'create' GROUP BY event ORDER BY count DESC;
+SELECT get_json_string(data, 'commit.collection') AS event, HOUR(from_microsecond(get_json_int(data, 'time_us'))) AS hour_of_day, COUNT(*) AS count FROM bluesky WHERE get_json_string(data, 'kind') = 'commit' AND get_json_string(data, 'commit.operation') = 'create' AND get_json_string(data, 'commit.collection') IN ('app.bsky.feed.post', 'app.bsky.feed.repost', 'app.bsky.feed.like') GROUP BY event, hour_of_day ORDER BY hour_of_day, event;
+SELECT get_json_string(data, 'did') AS user_id, MIN(from_microsecond(get_json_int(data, 'time_us'))) AS first_post_ts FROM bluesky WHERE get_json_string(data, 'kind') = 'commit' AND get_json_string(data, 'commit.operation') = 'create' AND get_json_string(data, 'commit.collection') = 'app.bsky.feed.post' GROUP BY user_id ORDER BY first_post_ts ASC LIMIT 3;
+SELECT get_json_string(data, 'did') AS user_id, MILLISECONDS_DIFF(MAX(from_microsecond(get_json_int(data, 'time_us'))),MIN(from_microsecond(get_json_int(data, 'time_us')))) AS activity_span FROM bluesky WHERE get_json_string(data, 'kind') = 'commit' AND get_json_string(data, 'commit.operation') = 'create' AND get_json_string(data, 'commit.collection') = 'app.bsky.feed.post' GROUP BY user_id ORDER BY activity_span DESC LIMIT 3;

doris/queries_default.sql

@@ -1,20 +1,21 @@
 #!/bin/bash
 
 # Check if the required arguments are provided
-if [[ $# -lt 1 ]]; then
-    echo "Usage: $0 <DB_NAME>"
+if [[ $# -lt 2 ]]; then
+    echo "Usage: $0 <DB_NAME> <QUERIES_FILE>"
     exit 1
 fi
 
 # Arguments
 DB_NAME="$1"
+QUERIES_FILE="$2"
 
 TRIES=3
 
 mysql -P 9030 -h 127.0.0.1 -u root $DB_NAME -e "set global parallel_pipeline_task_num=32;"
 mysql -P 9030 -h 127.0.0.1 -u root $DB_NAME -e "set global enable_parallel_scan=false;"
 
-cat queries.sql | while read -r query; do
+cat $QUERIES_FILE | while read -r query; do
 
     # Clear the Linux file system cache
     echo "Clearing file system cache..."

doris/queries.sql doris/queries_materialized.sqldoris/queries.sql renamed to doris/queries_materialized.sql

@@ -1,6 +1,4 @@
 #!/bin/bash
-export JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64
-
 ./${DORIS_PACKAGE}/be/bin/start_be.sh --daemon
 ./${DORIS_PACKAGE}/fe/bin/start_fe.sh --daemon