Merge pull request #175 from tembo-io/main

Tembo OLAP Stack Benchmark

Author: tinybit

index.html

@@ -432,6 +432,7 @@
 ,{"system":"SQLite","date":"2022-07-01","machine":"c6a.4xlarge, 500gb gp2","cluster_size":1,"comment":"","tags":["C","embedded","row-oriented"],"load_time":2608,"data_size":75776589824,"result":[[752.739,2.003,1.2],[304.302,291.521,286.965],[293.964,287.619,287.219],[758.302,5.879,5.65],[836.393,48.593,48.452],[362.605,344.884,356.245],[763.993,11.602,10.795],[296.348,286.879,287.557],[365.816,360.339,354.126],[374.403,365.196,362.261],[302.989,293.888,298.432],[303.64,291.729,295.347],[316.824,298.18,301.006],[320.665,301.15,305.227],[313.593,301.021,301.626],[794.881,47,47.225],[355.346,344.615,342.442],[316.499,305.971,305.007],[398.177,380.383,385.571],[751.82,5.082,4.913],[295.522,286.573,287.368],[298.58,287.182,288.303],[296.474,288.747,288.638],[296.579,287.127,287.361],[304.709,286.865,287.56],[300.391,290.633,288.587],[294.605,286.91,287.799],[305.986,312.111,305.626],[null,null,null],[411.225,397.614,394.253],[307.711,295.181,300.266],[312.472,299.079,298.19],[386.674,378.347,376.963],[409.742,409.554,420.273],[468.73,453.709,458.149],[366.015,347.446,346.728],[2.911,0.781,0.757],[1.599,0.609,0.587],[1.288,0.256,0.238],[2.469,1.582,1.52],[1.274,0.303,0.283],[1.322,0.317,0.314],[1.498,0.602,0.613]],"source":"sqlite/results/c6a.4xlarge.json"}
 ,{"system":"StarRocks","date":"2022-09-25","machine":"c6a.4xlarge, 500gb gp2","cluster_size":1,"comment":"","tags":["C++","column-oriented","MySQL compatible","ClickHouse derivative"],"load_time":492,"data_size":17233927256,"result":[[0.04,0.02,0.02],[0.21,0.03,0.04],[1.88,0.07,0.07],[3.4,0.09,0.1],[0.37,0.34,0.33],[2.71,0.88,0.84],[0.03,0.02,0.02],[0.06,0.03,0.03],[1.66,0.53,0.47],[1.58,1.52,1.47],[0.68,0.1,0.11],[1.26,0.12,0.12],[0.54,0.54,0.54],[0.99,0.96,0.99],[0.81,0.77,0.8],[0.41,0.39,0.4],[1.45,1.49,1.52],[0.52,0.52,0.51],[2.98,2.6,2.56],[0.02,0.01,0],[12.97,1.1,1.11],[14.84,0.91,0.94],[27.68,1.96,1.94],[46.49,1.49,1.22],[0.16,0.11,0.07],[0.94,0.12,0.12],[0.06,0.05,0.04],[13.99,1.44,1.42],[11.58,10.2,10.37],[0.22,0.1,0.11],[4.6,0.38,0.36],[4.15,0.53,0.52],[3.54,3.48,3.48],[15.22,6.08,6.17],[14.9,6.34,6.31],[1.86,0.59,0.51],[0.99,0.09,0.08],[0.42,0.05,0.05],[0.15,0.04,0.04],[1.27,0.18,0.18],[0.32,0.04,0.03],[0.11,0.03,0.04],[0.08,0.03,0.02]],"source":"starrocks/results/c6a.4xlarge.json"}
 ,{"system":"StarRocks","date":"2023-02-10","machine":"c6a.metal, 500gb gp2","cluster_size":1,"comment":"","tags":["C++","column-oriented","MySQL compatible","ClickHouse derivative"],"load_time":433,"data_size":17705230744,"result":[[0.04,0.04,0.03],[0.12,0.03,0.03],[0.66,0.06,0.06],[2.08,0.08,0.06],[0.1,0.08,0.09],[2.19,0.22,0.21],[0.04,0.02,0.03],[0.06,0.04,0.04],[0.99,0.12,0.12],[0.73,0.74,0.7],[0.13,0.06,0.05],[1.04,0.06,0.07],[0.17,0.18,0.17],[0.23,0.23,0.23],[0.54,0.18,0.18],[0.09,0.09,0.08],[0.31,0.32,0.3],[0.12,0.1,0.11],[1.27,0.48,0.44],[0.01,0.01,0],[12.09,0.17,0.15],[0.17,0.13,0.12],[10.91,0.28,0.24],[28.25,0.86,1.36],[0.03,0.03,0.02],[0.06,0.04,0.04],[0.02,0.03,0.02],[0.63,0.24,0.21],[8.77,1.6,1.64],[0.12,0.1,0.1],[1.33,0.13,0.13],[3.46,0.18,0.19],[0.97,0.86,0.86],[0.95,0.96,0.98],[0.96,0.97,0.97],[0.14,0.12,0.14],[0.07,0.06,0.06],[0.05,0.04,0.04],[0.04,0.03,0.03],[0.1,0.09,0.09],[0.47,0.03,0.03],[0.25,0.03,0.03],[0.04,0.03,0.03]],"source":"starrocks/results/c6a.metal.json"}
+,{"system":"Tembo OLAP(Columnar)","date":"2024-02-09","machine":"c6a.4xlarge, 300gb gp3","cluster_size":1,"comment":"","tags":["C","PostgreSQL compatible","column-oriented"],"load_time":14998.247,"data_size":33864704000,"result":[[0.322266,0.279012,0.278818],[0.262802,0.263067,0.263524],[1.03503,1.02851,1.01109],[0.694882,0.69194,0.684567],[24.1366,23.8622,24.2121],[37.8125,37.3546,37.4614],[0.088267,0.071204,0.072592],[0.27134,0.272216,0.270787],[25.1755,24.8466,24.944],[28.1199,28.1654,28.4494],[2.20623,2.17468,2.14656],[2.4656,2.4581,2.45926],[7.42677,7.57167,7.69995],[10.5288,10.3381,10.4471],[8.29726,8.19628,8.19725],[16.6094,16.9222,16.7514],[38.8504,38.4172,38.3965],[0.388963,0.358219,0.36013],[75.809,75.2281,76.712],[0.302681,0.303397,0.302277],[2.62566,2.61241,2.61167],[2.8029,2.81523,2.84338],[3.70515,3.68054,3.66689],[21.0447,20.8168,21.0621],[1.44812,1.34319,1.33672],[0.34479,0.318921,0.317873],[1.21699,1.20804,1.20271],[4.83795,4.79006,4.79372],[41.5645,41.008,40.9968],[7.96046,8.03584,7.98607],[8.77845,9.01301,8.9083],[14.8007,14.662,14.7183],[69.205,69.6116,70.0597],[32.5098,31.3654,32.3301],[33.5434,33.3958,33.0693],[14.3637,14.5175,14.395],[0.373303,0.373503,0.376101],[0.193498,0.190686,0.194189],[0.103831,0.111741,0.119202],[0.754824,0.760665,0.798245],[0.110722,0.110677,0.114796],[0.101987,0.10852,0.101449],[0.148887,0.12897,0.146846]],"source":"tembo-olap/results/tembo-olap-col-c6a.json"}
 ,{"hide":false,"system":"Tablespace","date":"2024-02-25","machine":"L1 - 16CPU 32GB","cluster_size":1,"comment":"","tags":["C","Rust","row-oriented","column-oriented","PostgreSQL compatible","managed","analytical"],"load_time":13711,"data_size":135469176422,"result":[[0.436542,0.032826,0.058116],[0.475804,0.06273,0.069842],[0.221264,0.118931,0.108054],[1.13058,0.110611,0.107611],[0.842296,0.840424,0.84895],[2.84863,1.22785,1.17402],[0.009753,0.002873,0.003417],[0.195597,0.098896,0.074905],[1.70796,1.37998,1.54147],[1.4772,1.33613,1.31476],[0.373194,0.340836,0.338413],[0.375037,0.361468,0.378387],[1.16898,1.22366,1.21796],[2.09798,2.94135,2.16265],[1.52902,1.34797,1.2899],[0.972571,0.981924,0.914636],[2.43485,2.58636,2.75724],[0.005437,0.003755,0.003427],[5.30082,5.16673,5.20788],[0.006782,0.002805,0.002702],[16.7202,0.179415,0.169258],[0.210038,0.180524,0.174665],[23.7864,0.2504,0.242016],[0.183456,0.173261,0.17237],[0.048444,0.003245,0.003059],[0.007258,0.003278,0.003245],[0.015761,0.003279,0.003193],[22.3888,1.79019,1.70748],[17.1564,4.59906,18.7535],[0.599159,0.572579,0.5331],[1.20788,1.13153,1.1528],[3.71611,1.39238,1.40143],[8.40708,8.20716,7.2647],[46.2087,5.43577,45.5534],[6.23327,6.50139,6.08041],[2.44898,2.23633,2.08641],[0.42763,0.352619,0.30957],[0.678261,0.476425,0.445909],[0.179801,0.154265,0.160855],[0.776884,0.791492,0.785599],[0.171634,0.11076,0.090103],[0.110994,0.084385,0.084405],[0.103362,0.10878,0.093837]],"source":"tablespace/results/large-1.json"}
 ,{"system":"TimescaleDB (compression)","date":"2022-07-01","machine":"c6a.4xlarge, 500gb gp2","cluster_size":1,"comment":"","tags":["C","PostgreSQL compatible","column-oriented","time-series"],"load_time":4605,"data_size":20333747165,"result":[[2.28686,1.63642,1.64263],[32.6848,1.63476,1.40052],[60.8633,3.70484,3.59342],[36.4029,2.87091,2.80739],[110.391,38.9688,38.0549],[147.379,66.2513,65.6379],[33.0294,2.92031,2.84375],[33.0221,1.2984,1.19227],[115.694,47.4651,47.0125],[156.347,51.577,51.2694],[68.3301,4.75521,4.68007],[77.4356,5.55128,5.56577],[49.7741,11.2911,11.3265],[81.1014,14.9111,14.9541],[82.9569,14.6156,14.6331],[62.0338,26.399,26.3351],[103.259,36.4122,36.6076],[92.8828,26.2395,25.8991],[144.281,63.5102,63.7661],[7.00679,0.573073,0.536283],[75.0203,7.86344,7.90495],[81.2825,9.15868,9.01775],[104.084,13.9528,13.8435],[132.531,81.522,82.1561],[80.6965,3.28231,3.16574],[39.7693,2.51443,2.43849],[80.4245,3.26941,3.13916],[104.015,13.7044,13.5313],[307.26,253.127,252.147],[42.8549,22.4187,22.0325],[137.601,14.9592,14.6804],[136.767,22.8007,22.131],[263.005,168.551,163.355],[156.919,92.6308,91.702],[160.842,96.0512,97.1773],[62.8357,28.0336,28.7397],[1.75869,0.561604,0.541215],[0.46607,0.191863,0.19021],[0.303671,0.137579,0.136615],[2.32031,1.49223,1.52369],[0.563764,0.14192,0.138234],[0.372428,0.122989,0.123709],[0.448574,0.159092,0.154687]],"source":"timescaledb-compressed/results/c6a.4xlarge.json"}
 ,{"system":"TimescaleDB","date":"2022-07-01","machine":"c6a.4xlarge, 500gb gp2","cluster_size":1,"comment":"","tags":["C","PostgreSQL compatible","row-oriented","time-series"],"load_time":1620,"data_size":72882392030,"result":[[437.7,215.793,176.42],[327.026,259.568,244.578],[262.978,263.09,263.083],[262.807,263.046,266.847],[337.497,334.964,330.852],[355.689,356.801,362.894],[262.762,263.012,262.968],[263.055,263.016,263.028],[319.928,319.388,320.704],[323.584,322.224,322.488],[265.979,265.465,265.375],[266.019,265.543,265.462],[277.018,276.3,276.595],[280.352,279.251,279.572],[279.915,279.896,279.674],[296.377,298.506,297.659],[314.448,314.605,312.57],[302.668,302.672,303.039],[325.81,324.061,324.376],[262.447,262.698,262.704],[267.581,267.467,267.482],[268.085,267.466,267.696],[263.391,263.097,263.126],[38.291,0.435,0.335],[0.127,0.005,0.005],[263.138,263.1,263.092],[0.889,0.341,0.339],[267.586,267.498,267.491],[289.086,290.012,290.093],[263.22,263.071,263.109],[274.78,273.995,273.998],[282.217,281.39,281.47],[429.273,426.588,439.431],[448.808,418.724,418.207],[455.196,422.75,423.142],[299.263,296.937,297.261],[18.693,1.552,1.481],[18.125,0.945,0.937],[18.528,1.062,0.902],[18.268,1.779,1.77],[19.615,1.965,1.966],[18.97,1.435,1.43],[18.33,1.153,0.952]],"source":"timescaledb/results/c6a.4xlarge.json"}

tembo-olap/README.md

@@ -0,0 +1,31 @@
+# Instantiation
+
+1) Create Account and Login to https://cloud.tembo.io
+2) Select OLAP Stack after clicking Create Instance button
+
+## Choose System Configuration:
+
+**Instance Type**: Compute Optimized (CI-32)
+
+**CPU**: 16vCPU
+
+**RAM** : 32GB
+
+**Storage**: 500GB 
+
+Note: instance creation takes *20-25* seconds.
+
+## Specify Connection params
+Tembo provides connection-string through instance **Home** tab.
+
+Use hostname and password taken from Tembo console and specify them in benchmark.sh
+```
+HOSTNAME="<hostname>"
+PASSWORD="<password>"
+```
+
+## Start Benchmark
+
+```
+./benchmark.sh
+```

tembo-olap/benchmark.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+HOSTNAME="<hostname>"
+PASSWORD="<password>"
+
+sudo apt-get update
+sudo apt-get install -y postgresql-client
+
+wget --no-verbose --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+gzip -d hits.tsv.gz
+chmod 777 ~ hits.tsv
+
+psql postgresql://postgres:$PASSWORD@$HOSTNAME:5432 -t -c 'CREATE DATABASE test'
+psql "host=$HOSTNAME port=5432 dbname=test user=postgres password=$PASSWORD sslmode=require" < create.sql
+psql "host=$HOSTNAME port=5432 dbname=test user=postgres password=$PASSWORD sslmode=require" -t -c '\timing' -c "\\copy hits FROM 'hits.tsv'"
+psql "host=$HOSTNAME port=5432 dbname=test user=postgres password=$PASSWORD sslmode=require" < index.sql
+psql "host=$HOSTNAME port=5432 dbname=test user=postgres password=$PASSWORD sslmode=require" -t -c '\timing' -c "select pg_total_relation_size('hits');"
+
+./run.sh "${HOSTNAME}" "${PASSWORD}" 2>&1 | tee log.txt
+
+cat log.txt |
+grep -oP 'Time: \d+\.\d+ ms' |
+sed -r -e 's/Time: ([0-9]+\.[0-9]+) ms/\1/' |
+awk '{
+    if (i % 3 == 0) {
+        printf "[";
+    }
+    printf $1 / 1000;
+    if (i % 3 != 2) {
+        printf ",";
+    } else {
+        print "],";
+    }
+    ++i;
+}'

tembo-olap/create.sql

@@ -0,0 +1,112 @@
+CREATE EXTENSION IF NOT EXISTS COLUMNAR;
+DROP TABLE IF EXISTS hits CASCADE;
+CREATE TABLE hits
+(
+    WatchID BIGINT NOT NULL,
+    JavaEnable SMALLINT NOT NULL,
+    Title TEXT NOT NULL,
+    GoodEvent SMALLINT NOT NULL,
+    EventTime TIMESTAMP NOT NULL,
+    EventDate Date NOT NULL,
+    CounterID INTEGER NOT NULL,
+    ClientIP INTEGER NOT NULL,
+    RegionID INTEGER NOT NULL,
+    UserID BIGINT NOT NULL,
+    CounterClass SMALLINT NOT NULL,
+    OS SMALLINT NOT NULL,
+    UserAgent SMALLINT NOT NULL,
+    URL TEXT NOT NULL,
+    Referer TEXT NOT NULL,
+    IsRefresh SMALLINT NOT NULL,
+    RefererCategoryID SMALLINT NOT NULL,
+    RefererRegionID INTEGER NOT NULL,
+    URLCategoryID SMALLINT NOT NULL,
+    URLRegionID INTEGER NOT NULL,
+    ResolutionWidth SMALLINT NOT NULL,
+    ResolutionHeight SMALLINT NOT NULL,
+    ResolutionDepth SMALLINT NOT NULL,
+    FlashMajor SMALLINT NOT NULL,
+    FlashMinor SMALLINT NOT NULL,
+    FlashMinor2 TEXT NOT NULL,
+    NetMajor SMALLINT NOT NULL,
+    NetMinor SMALLINT NOT NULL,
+    UserAgentMajor SMALLINT NOT NULL,
+    UserAgentMinor VARCHAR(255) NOT NULL,
+    CookieEnable SMALLINT NOT NULL,
+    JavascriptEnable SMALLINT NOT NULL,
+    IsMobile SMALLINT NOT NULL,
+    MobilePhone SMALLINT NOT NULL,
+    MobilePhoneModel TEXT NOT NULL,
+    Params TEXT NOT NULL,
+    IPNetworkID INTEGER NOT NULL,
+    TraficSourceID SMALLINT NOT NULL,
+    SearchEngineID SMALLINT NOT NULL,
+    SearchPhrase TEXT NOT NULL,
+    AdvEngineID SMALLINT NOT NULL,
+    IsArtifical SMALLINT NOT NULL,
+    WindowClientWidth SMALLINT NOT NULL,
+    WindowClientHeight SMALLINT NOT NULL,
+    ClientTimeZone SMALLINT NOT NULL,
+    ClientEventTime TIMESTAMP NOT NULL,
+    SilverlightVersion1 SMALLINT NOT NULL,
+    SilverlightVersion2 SMALLINT NOT NULL,
+    SilverlightVersion3 INTEGER NOT NULL,
+    SilverlightVersion4 SMALLINT NOT NULL,
+    PageCharset TEXT NOT NULL,
+    CodeVersion INTEGER NOT NULL,
+    IsLink SMALLINT NOT NULL,
+    IsDownload SMALLINT NOT NULL,
+    IsNotBounce SMALLINT NOT NULL,
+    FUniqID BIGINT NOT NULL,
+    OriginalURL TEXT NOT NULL,
+    HID INTEGER NOT NULL,
+    IsOldCounter SMALLINT NOT NULL,
+    IsEvent SMALLINT NOT NULL,
+    IsParameter SMALLINT NOT NULL,
+    DontCountHits SMALLINT NOT NULL,
+    WithHash SMALLINT NOT NULL,
+    HitColor CHAR NOT NULL,
+    LocalEventTime TIMESTAMP NOT NULL,
+    Age SMALLINT NOT NULL,
+    Sex SMALLINT NOT NULL,
+    Income SMALLINT NOT NULL,
+    Interests SMALLINT NOT NULL,
+    Robotness SMALLINT NOT NULL,
+    RemoteIP INTEGER NOT NULL,
+    WindowName INTEGER NOT NULL,
+    OpenerName INTEGER NOT NULL,
+    HistoryLength SMALLINT NOT NULL,
+    BrowserLanguage TEXT NOT NULL,
+    BrowserCountry TEXT NOT NULL,
+    SocialNetwork TEXT NOT NULL,
+    SocialAction TEXT NOT NULL,
+    HTTPError SMALLINT NOT NULL,
+    SendTiming INTEGER NOT NULL,
+    DNSTiming INTEGER NOT NULL,
+    ConnectTiming INTEGER NOT NULL,
+    ResponseStartTiming INTEGER NOT NULL,
+    ResponseEndTiming INTEGER NOT NULL,
+    FetchTiming INTEGER NOT NULL,
+    SocialSourceNetworkID SMALLINT NOT NULL,
+    SocialSourcePage TEXT NOT NULL,
+    ParamPrice BIGINT NOT NULL,
+    ParamOrderID TEXT NOT NULL,
+    ParamCurrency TEXT NOT NULL,
+    ParamCurrencyID SMALLINT NOT NULL,
+    OpenstatServiceName TEXT NOT NULL,
+    OpenstatCampaignID TEXT NOT NULL,
+    OpenstatAdID TEXT NOT NULL,
+    OpenstatSourceID TEXT NOT NULL,
+    UTMSource TEXT NOT NULL,
+    UTMMedium TEXT NOT NULL,
+    UTMCampaign TEXT NOT NULL,
+    UTMContent TEXT NOT NULL,
+    UTMTerm TEXT NOT NULL,
+    FromTag TEXT NOT NULL,
+    HasGCLID SMALLINT NOT NULL,
+    RefererHash BIGINT NOT NULL,
+    URLHash BIGINT NOT NULL,
+    CLID INTEGER NOT NULL,
+    PRIMARY KEY (CounterID, EventDate, UserID, EventTime, WatchID)
+)USING columnar;
+

tembo-olap/index.sql

@@ -0,0 +1,18 @@
+CREATE INDEX adveng on hits (AdvEngineID);
+CREATE INDEX regid  on hits (RegionID);
+CREATE INDEX cid on hits (CounterID);
+CREATE INDEX eventtime on hits (EventTime);
+CREATE INDEX eventdate on hits (EventDate);
+CREATE INDEX mobile on hits (MobilePhoneModel);
+CREATE INDEX refresh on hits (IsRefresh, DontCountHits);
+CREATE INDEX resolutionwidth on hits (ResolutionWidth);
+CREATE INDEX search on hits (SearchPhrase);
+CREATE INDEX UserID on hits (UserID);
+
+CREATE INDEX UserIDsearch on hits (UserID, SearchPhrase);
+CREATE INDEX widcip on hits (WatchID, ClientIP);
+CREATE INDEX mobileuser on hits (MobilePhoneModel,UserID);
+CREATE INDEX regionuser on hits (RegionID,UserID);
+
+CREATE INDEX mobile2 on hits (MobilePhoneModel) WHERE MobilePhoneModel <> ''::text;
+CREATE INDEX search2 on hits (SearchPhrase) WHERE SearchPhrase <> ''::text;