update

Author: BiteTheDDDDt

doris/ddl_default_with_index.sql

@@ -0,0 +1,9 @@
+CREATE TABLE bluesky (
+    `id` BIGINT NOT NULL AUTO_INCREMENT,
+    `data` variant NOT NULL,
+    INDEX idx_var (`data`) USING INVERTED
+)
+DISTRIBUTED BY HASH(id) BUCKETS 32
+PROPERTIES (
+    "replication_num"="1"
+);

doris/main.sh

@@ -58,29 +58,36 @@ benchmark() {
 case $CHOICE in
     2)
         benchmark 10 default
+        benchmark 10 default_with_index
         benchmark 10 materialized
         ;;
     3)
         benchmark 100 default
+        benchmark 100 default_with_index
         benchmark 100 materialized
         ;;
     4)
         benchmark 1000 default
+        benchmark 1000 default_with_index
         benchmark 1000 materialized
         ;;
     5)
-        benchmark 1 materialized
         benchmark 1 default
-        benchmark 10 materialized
+        benchmark 1 default_with_index
+        benchmark 1 materialized
         benchmark 10 default
-        benchmark 100 materialized
+        benchmark 10 default_with_index
+        benchmark 10 materialized
         benchmark 100 default
+        benchmark 100 default_with_index
+        benchmark 100 materialized
+        benchmark 1000 default_with_index
         benchmark 1000 materialized
-        benchmark 1000 default
         ;;
     *)
-        benchmark 1 materialized
         benchmark 1 default
+        benchmark 1 default_with_index
+        benchmark 1 materialized
         ;;
 esac
 

doris/queries_default_with_index.sql

@@ -0,0 +1,5 @@
+SELECT cast(data['commit']['collection'] AS TEXT ) AS event, COUNT(*) AS count FROM bluesky GROUP BY event ORDER BY count DESC;
+SELECT cast(data['commit']['collection'] AS TEXT ) AS event, COUNT(*) AS count, COUNT(DISTINCT cast(data['did'] AS TEXT )) AS users FROM bluesky WHERE cast(data['kind'] AS TEXT ) = 'commit' AND cast(data['commit']['operation'] AS TEXT ) = 'create' GROUP BY event ORDER BY count DESC;
+SELECT cast(data['commit']['collection'] AS TEXT ) AS event, HOUR(from_microsecond(CAST(data['time_us'] AS BIGINT))) AS hour_of_day, COUNT(*) AS count FROM bluesky WHERE cast(data['kind'] AS TEXT ) = 'commit' AND cast(data['commit']['operation'] AS TEXT ) = 'create' AND cast(data['commit']['collection'] AS TEXT ) IN ('app.bsky.feed.post', 'app.bsky.feed.repost', 'app.bsky.feed.like') GROUP BY event, hour_of_day ORDER BY hour_of_day, event;
+SELECT cast(data['did'] AS TEXT ) AS user_id, MIN(from_microsecond(CAST(data['time_us'] AS BIGINT))) AS first_post_ts FROM bluesky WHERE cast(data['kind'] AS TEXT ) = 'commit' AND cast(data['commit']['operation'] AS TEXT ) = 'create' AND cast(data['commit']['collection'] AS TEXT ) = 'app.bsky.feed.post' GROUP BY user_id ORDER BY first_post_ts ASC LIMIT 3;
+SELECT cast(data['did'] AS TEXT ) AS user_id, MILLISECONDS_DIFF(MAX(from_microsecond(CAST(data['time_us'] AS BIGINT))),MIN(from_microsecond(CAST(data['time_us'] AS BIGINT)))) AS activity_span FROM bluesky WHERE cast(data['kind'] AS TEXT ) = 'commit' AND cast(data['commit']['operation'] AS TEXT ) = 'create' AND cast(data['commit']['collection'] AS TEXT ) = 'app.bsky.feed.post' GROUP BY user_id ORDER BY activity_span DESC LIMIT 3;

doris/results/m6i.8xlarge_bluesky_1000m_default.json

@@ -1,8 +1,8 @@
 #!/bin/bash
 
 export JAVA_HOME="/usr/lib/jvm/java-17-openjdk-amd64"
-sysctl -w vm.max_map_count=2000000
-ulimit -n 655350
+sudo sysctl -w vm.max_map_count=2000000
+sudo ulimit -n 655350
 
 ${DORIS_FULL_NAME}/be/bin/start_be.sh --daemon
 ${DORIS_FULL_NAME}/fe/bin/start_fe.sh --daemon