Tembo OLAP Stack Benchmark

Author: shhnwz

index.html

@@ -431,6 +431,7 @@
 ,{"system":"SQLite","date":"2022-07-01","machine":"c6a.4xlarge, 500gb gp2","cluster_size":1,"comment":"","tags":["C","embedded","row-oriented"],"load_time":2608,"data_size":75776589824,"result":[[752.739,2.003,1.2],[304.302,291.521,286.965],[293.964,287.619,287.219],[758.302,5.879,5.65],[836.393,48.593,48.452],[362.605,344.884,356.245],[763.993,11.602,10.795],[296.348,286.879,287.557],[365.816,360.339,354.126],[374.403,365.196,362.261],[302.989,293.888,298.432],[303.64,291.729,295.347],[316.824,298.18,301.006],[320.665,301.15,305.227],[313.593,301.021,301.626],[794.881,47,47.225],[355.346,344.615,342.442],[316.499,305.971,305.007],[398.177,380.383,385.571],[751.82,5.082,4.913],[295.522,286.573,287.368],[298.58,287.182,288.303],[296.474,288.747,288.638],[296.579,287.127,287.361],[304.709,286.865,287.56],[300.391,290.633,288.587],[294.605,286.91,287.799],[305.986,312.111,305.626],[null,null,null],[411.225,397.614,394.253],[307.711,295.181,300.266],[312.472,299.079,298.19],[386.674,378.347,376.963],[409.742,409.554,420.273],[468.73,453.709,458.149],[366.015,347.446,346.728],[2.911,0.781,0.757],[1.599,0.609,0.587],[1.288,0.256,0.238],[2.469,1.582,1.52],[1.274,0.303,0.283],[1.322,0.317,0.314],[1.498,0.602,0.613]],"source":"sqlite/results/c6a.4xlarge.json"}
 ,{"system":"StarRocks","date":"2022-09-25","machine":"c6a.4xlarge, 500gb gp2","cluster_size":1,"comment":"","tags":["C++","column-oriented","MySQL compatible","ClickHouse derivative"],"load_time":492,"data_size":17233927256,"result":[[0.04,0.02,0.02],[0.21,0.03,0.04],[1.88,0.07,0.07],[3.4,0.09,0.1],[0.37,0.34,0.33],[2.71,0.88,0.84],[0.03,0.02,0.02],[0.06,0.03,0.03],[1.66,0.53,0.47],[1.58,1.52,1.47],[0.68,0.1,0.11],[1.26,0.12,0.12],[0.54,0.54,0.54],[0.99,0.96,0.99],[0.81,0.77,0.8],[0.41,0.39,0.4],[1.45,1.49,1.52],[0.52,0.52,0.51],[2.98,2.6,2.56],[0.02,0.01,0],[12.97,1.1,1.11],[14.84,0.91,0.94],[27.68,1.96,1.94],[46.49,1.49,1.22],[0.16,0.11,0.07],[0.94,0.12,0.12],[0.06,0.05,0.04],[13.99,1.44,1.42],[11.58,10.2,10.37],[0.22,0.1,0.11],[4.6,0.38,0.36],[4.15,0.53,0.52],[3.54,3.48,3.48],[15.22,6.08,6.17],[14.9,6.34,6.31],[1.86,0.59,0.51],[0.99,0.09,0.08],[0.42,0.05,0.05],[0.15,0.04,0.04],[1.27,0.18,0.18],[0.32,0.04,0.03],[0.11,0.03,0.04],[0.08,0.03,0.02]],"source":"starrocks/results/c6a.4xlarge.json"}
 ,{"system":"StarRocks","date":"2023-02-10","machine":"c6a.metal, 500gb gp2","cluster_size":1,"comment":"","tags":["C++","column-oriented","MySQL compatible","ClickHouse derivative"],"load_time":433,"data_size":17705230744,"result":[[0.04,0.04,0.03],[0.12,0.03,0.03],[0.66,0.06,0.06],[2.08,0.08,0.06],[0.1,0.08,0.09],[2.19,0.22,0.21],[0.04,0.02,0.03],[0.06,0.04,0.04],[0.99,0.12,0.12],[0.73,0.74,0.7],[0.13,0.06,0.05],[1.04,0.06,0.07],[0.17,0.18,0.17],[0.23,0.23,0.23],[0.54,0.18,0.18],[0.09,0.09,0.08],[0.31,0.32,0.3],[0.12,0.1,0.11],[1.27,0.48,0.44],[0.01,0.01,0],[12.09,0.17,0.15],[0.17,0.13,0.12],[10.91,0.28,0.24],[28.25,0.86,1.36],[0.03,0.03,0.02],[0.06,0.04,0.04],[0.02,0.03,0.02],[0.63,0.24,0.21],[8.77,1.6,1.64],[0.12,0.1,0.1],[1.33,0.13,0.13],[3.46,0.18,0.19],[0.97,0.86,0.86],[0.95,0.96,0.98],[0.96,0.97,0.97],[0.14,0.12,0.14],[0.07,0.06,0.06],[0.05,0.04,0.04],[0.04,0.03,0.03],[0.1,0.09,0.09],[0.47,0.03,0.03],[0.25,0.03,0.03],[0.04,0.03,0.03]],"source":"starrocks/results/c6a.metal.json"}
+,{"system":"Tembo OLAP(Columnar)","date":"2024-02-09","machine":"c6a.4xlarge, 300gb gp3","cluster_size":1,"comment":"","tags":["C","PostgreSQL compatible","column-oriented"],"load_time":14998.247,"data_size":33864704000,"result":[[0.322266,0.279012,0.278818],[0.262802,0.263067,0.263524],[1.03503,1.02851,1.01109],[0.694882,0.69194,0.684567],[24.1366,23.8622,24.2121],[37.8125,37.3546,37.4614],[0.088267,0.071204,0.072592],[0.27134,0.272216,0.270787],[25.1755,24.8466,24.944],[28.1199,28.1654,28.4494],[2.20623,2.17468,2.14656],[2.4656,2.4581,2.45926],[7.42677,7.57167,7.69995],[10.5288,10.3381,10.4471],[8.29726,8.19628,8.19725],[16.6094,16.9222,16.7514],[38.8504,38.4172,38.3965],[0.388963,0.358219,0.36013],[75.809,75.2281,76.712],[0.302681,0.303397,0.302277],[2.62566,2.61241,2.61167],[2.8029,2.81523,2.84338],[3.70515,3.68054,3.66689],[21.0447,20.8168,21.0621],[1.44812,1.34319,1.33672],[0.34479,0.318921,0.317873],[1.21699,1.20804,1.20271],[4.83795,4.79006,4.79372],[41.5645,41.008,40.9968],[7.96046,8.03584,7.98607],[8.77845,9.01301,8.9083],[14.8007,14.662,14.7183],[69.205,69.6116,70.0597],[32.5098,31.3654,32.3301],[33.5434,33.3958,33.0693],[14.3637,14.5175,14.395],[0.373303,0.373503,0.376101],[0.193498,0.190686,0.194189],[0.103831,0.111741,0.119202],[0.754824,0.760665,0.798245],[0.110722,0.110677,0.114796],[0.101987,0.10852,0.101449],[0.148887,0.12897,0.146846]],"source":"tembo-olap/results/tembo-olap-col-c6a.json"}
 ,{"system":"TimescaleDB (compression)","date":"2022-07-01","machine":"c6a.4xlarge, 500gb gp2","cluster_size":1,"comment":"","tags":["C","PostgreSQL compatible","column-oriented","time-series"],"load_time":4605,"data_size":20333747165,"result":[[2.28686,1.63642,1.64263],[32.6848,1.63476,1.40052],[60.8633,3.70484,3.59342],[36.4029,2.87091,2.80739],[110.391,38.9688,38.0549],[147.379,66.2513,65.6379],[33.0294,2.92031,2.84375],[33.0221,1.2984,1.19227],[115.694,47.4651,47.0125],[156.347,51.577,51.2694],[68.3301,4.75521,4.68007],[77.4356,5.55128,5.56577],[49.7741,11.2911,11.3265],[81.1014,14.9111,14.9541],[82.9569,14.6156,14.6331],[62.0338,26.399,26.3351],[103.259,36.4122,36.6076],[92.8828,26.2395,25.8991],[144.281,63.5102,63.7661],[7.00679,0.573073,0.536283],[75.0203,7.86344,7.90495],[81.2825,9.15868,9.01775],[104.084,13.9528,13.8435],[132.531,81.522,82.1561],[80.6965,3.28231,3.16574],[39.7693,2.51443,2.43849],[80.4245,3.26941,3.13916],[104.015,13.7044,13.5313],[307.26,253.127,252.147],[42.8549,22.4187,22.0325],[137.601,14.9592,14.6804],[136.767,22.8007,22.131],[263.005,168.551,163.355],[156.919,92.6308,91.702],[160.842,96.0512,97.1773],[62.8357,28.0336,28.7397],[1.75869,0.561604,0.541215],[0.46607,0.191863,0.19021],[0.303671,0.137579,0.136615],[2.32031,1.49223,1.52369],[0.563764,0.14192,0.138234],[0.372428,0.122989,0.123709],[0.448574,0.159092,0.154687]],"source":"timescaledb-compressed/results/c6a.4xlarge.json"}
 ,{"system":"TimescaleDB","date":"2022-07-01","machine":"c6a.4xlarge, 500gb gp2","cluster_size":1,"comment":"","tags":["C","PostgreSQL compatible","row-oriented","time-series"],"load_time":1620,"data_size":72882392030,"result":[[437.7,215.793,176.42],[327.026,259.568,244.578],[262.978,263.09,263.083],[262.807,263.046,266.847],[337.497,334.964,330.852],[355.689,356.801,362.894],[262.762,263.012,262.968],[263.055,263.016,263.028],[319.928,319.388,320.704],[323.584,322.224,322.488],[265.979,265.465,265.375],[266.019,265.543,265.462],[277.018,276.3,276.595],[280.352,279.251,279.572],[279.915,279.896,279.674],[296.377,298.506,297.659],[314.448,314.605,312.57],[302.668,302.672,303.039],[325.81,324.061,324.376],[262.447,262.698,262.704],[267.581,267.467,267.482],[268.085,267.466,267.696],[263.391,263.097,263.126],[38.291,0.435,0.335],[0.127,0.005,0.005],[263.138,263.1,263.092],[0.889,0.341,0.339],[267.586,267.498,267.491],[289.086,290.012,290.093],[263.22,263.071,263.109],[274.78,273.995,273.998],[282.217,281.39,281.47],[429.273,426.588,439.431],[448.808,418.724,418.207],[455.196,422.75,423.142],[299.263,296.937,297.261],[18.693,1.552,1.481],[18.125,0.945,0.937],[18.528,1.062,0.902],[18.268,1.779,1.77],[19.615,1.965,1.966],[18.97,1.435,1.43],[18.33,1.153,0.952]],"source":"timescaledb/results/c6a.4xlarge.json"}
 ,{"system":"Umbra","date":"2024-02-02","machine":"c6a.4xlarge, 500gb gp2","cluster_size":1,"tags":["C++","column-oriented","PostgreSQL compatible"],"load_time":1061.541,"data_size":97767877183,"comment":"Segmentation fault in the middle of the test","result":[[0.209086,0.011738,0.012061],[0.286187,0.011852,0.003785],[0.295367,0.135114,0.044054],[0.895806,0.679204,0.680544],[0.702135,0.674911,0.677621],[4.0023,0.87053,0.613472],[0.11274,0.078905,0.087947],[0.012837,0.014259,0.014031],[2.729,0.350596,0.341228],[0.696983,0.639588,0.669598],[0.339289,0.038563,0.043669],[0.209976,0.05442,0.054552],[0.430673,0.587625,0.768441],[0.932235,0.787055,0.780889],[1.28999,0.276156,0.297337],[0.202715,0.188756,0.192702],[0.530036,0.623375,0.608413],[0.511773,0.595728,0.563421],[1.42062,1.23585,1.192],[0.035434,0.013534,0.002076],[27.6449,14.838,12.4979],[11.6779,6.98523,4.57714],[28.1569,20.2952,10.3134],[153.316,153.2,154.629],[4.08712,1.57023,0.007798],[0.010185,0.010279,0.009956],[0.038879,0.038326,0.039015],[19.3634,7.89121,4.27973],[16.7256,10.2004,3.92077],[0.303641,0.054993,0.055398],[4.58902,0.332033,0.130684],[2.79384,1.48084,0.731322],[1.38139,1.6159,1.39411],[12.9291,9.05383,6.94245],[5.3304,5.41123,5.38231],[0.57384,0.254843,0.26153],[null,null,null],[null,null,null],[null,null,null],[null,null,null],[null,null,null],[null,null,null],[null,null,null]],"source":"umbra/results/c6a.4xlarge.json"}

tembo-olap/README.md

@@ -0,0 +1,76 @@
+# Instantiation
+
+1- Create Account and Login to https://cloud.tembo.io
+
+2- Select OLAP Stack after clicking Create Instance button.
+
+![Instance for benchmark](olap-stack.png "OLAP Stack")
+
+
+## Choose System Configuration:
+
+**Instance Type**: Compute Optimized (CI-32)
+
+**CPU**: 16vCPU
+
+**RAM** : 32GB
+
+**Storage**: 300GB
+
+
+![Instance Type](olap-instance-type.png "OLAP Stack")
+
+Instance creation takes *20-25* seconds.
+
+## Connection String
+Tembo provides connection-string through instance **Home** tab.
+
+```
+export PGPASSWORD=TEMBO_GENERATED_PGPASSWORD
+export PGHOST=TEMBO_GENERATED_PGHOST
+```
+
+## Data loading
+Data loaded remotely using psql client.
+
+**psql** installation
+
+```
+sudo apt-get update
+sudo apt-get install -y postgresql-client
+
+```
+**Data** download
+
+```
+wget --no-verbose --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+
+gzip -d hits.tsv.gz
+```
+
+**Data** loading
+
+```
+psql postgresql://postgres:$PGPASSWORD@$PGHOST:5432 -t -c 'CREATE DATABASE test'
+
+psql postgresql://postgres:$PGPASSWORD@$PGHOST:5432/test -t < create.sql
+
+psql postgres://postgres:$PGPASSWORD@$PGHOST:5432/test -t -c '\timing' -c "\\copy hits FROM 'hits.tsv'"
+
+psql postgresql://postgres:$PGPASSWORD@$PGHOST:5432/test -t < index.sql
+
+psql postgres://postgres:$PGPASSWORD@$PGHOST:5432/test -t -c "select pg_total_relation_size('hits');"
+            33864704000
+
+
+```
+
+## Benchmark Start
+
+```
+./run.sh 2>&1 | tee log.txt
+
+cat log.txt | grep -oP 'Time: \d+\.\d+ ms' | sed -r -e 's/Time: ([0-9]+\.[0-9]+) ms/\1/' |
+    awk '{ if (i % 3 == 0) { printf "[" }; printf $1 / 1000; if (i % 3 != 2) { printf "," } else { print "]," }; ++i; }'
+
+```

tembo-olap/create.sql

@@ -0,0 +1,112 @@
+CREATE EXTENSION IF NOT EXISTS COLUMNAR;
+DROP TABLE hits CASCADE;
+CREATE TABLE hits
+(
+    WatchID BIGINT NOT NULL,
+    JavaEnable SMALLINT NOT NULL,
+    Title TEXT NOT NULL,
+    GoodEvent SMALLINT NOT NULL,
+    EventTime TIMESTAMP NOT NULL,
+    EventDate Date NOT NULL,
+    CounterID INTEGER NOT NULL,
+    ClientIP INTEGER NOT NULL,
+    RegionID INTEGER NOT NULL,
+    UserID BIGINT NOT NULL,
+    CounterClass SMALLINT NOT NULL,
+    OS SMALLINT NOT NULL,
+    UserAgent SMALLINT NOT NULL,
+    URL TEXT NOT NULL,
+    Referer TEXT NOT NULL,
+    IsRefresh SMALLINT NOT NULL,
+    RefererCategoryID SMALLINT NOT NULL,
+    RefererRegionID INTEGER NOT NULL,
+    URLCategoryID SMALLINT NOT NULL,
+    URLRegionID INTEGER NOT NULL,
+    ResolutionWidth SMALLINT NOT NULL,
+    ResolutionHeight SMALLINT NOT NULL,
+    ResolutionDepth SMALLINT NOT NULL,
+    FlashMajor SMALLINT NOT NULL,
+    FlashMinor SMALLINT NOT NULL,
+    FlashMinor2 TEXT NOT NULL,
+    NetMajor SMALLINT NOT NULL,
+    NetMinor SMALLINT NOT NULL,
+    UserAgentMajor SMALLINT NOT NULL,
+    UserAgentMinor VARCHAR(255) NOT NULL,
+    CookieEnable SMALLINT NOT NULL,
+    JavascriptEnable SMALLINT NOT NULL,
+    IsMobile SMALLINT NOT NULL,
+    MobilePhone SMALLINT NOT NULL,
+    MobilePhoneModel TEXT NOT NULL,
+    Params TEXT NOT NULL,
+    IPNetworkID INTEGER NOT NULL,
+    TraficSourceID SMALLINT NOT NULL,
+    SearchEngineID SMALLINT NOT NULL,
+    SearchPhrase TEXT NOT NULL,
+    AdvEngineID SMALLINT NOT NULL,
+    IsArtifical SMALLINT NOT NULL,
+    WindowClientWidth SMALLINT NOT NULL,
+    WindowClientHeight SMALLINT NOT NULL,
+    ClientTimeZone SMALLINT NOT NULL,
+    ClientEventTime TIMESTAMP NOT NULL,
+    SilverlightVersion1 SMALLINT NOT NULL,
+    SilverlightVersion2 SMALLINT NOT NULL,
+    SilverlightVersion3 INTEGER NOT NULL,
+    SilverlightVersion4 SMALLINT NOT NULL,
+    PageCharset TEXT NOT NULL,
+    CodeVersion INTEGER NOT NULL,
+    IsLink SMALLINT NOT NULL,
+    IsDownload SMALLINT NOT NULL,
+    IsNotBounce SMALLINT NOT NULL,
+    FUniqID BIGINT NOT NULL,
+    OriginalURL TEXT NOT NULL,
+    HID INTEGER NOT NULL,
+    IsOldCounter SMALLINT NOT NULL,
+    IsEvent SMALLINT NOT NULL,
+    IsParameter SMALLINT NOT NULL,
+    DontCountHits SMALLINT NOT NULL,
+    WithHash SMALLINT NOT NULL,
+    HitColor CHAR NOT NULL,
+    LocalEventTime TIMESTAMP NOT NULL,
+    Age SMALLINT NOT NULL,
+    Sex SMALLINT NOT NULL,
+    Income SMALLINT NOT NULL,
+    Interests SMALLINT NOT NULL,
+    Robotness SMALLINT NOT NULL,
+    RemoteIP INTEGER NOT NULL,
+    WindowName INTEGER NOT NULL,
+    OpenerName INTEGER NOT NULL,
+    HistoryLength SMALLINT NOT NULL,
+    BrowserLanguage TEXT NOT NULL,
+    BrowserCountry TEXT NOT NULL,
+    SocialNetwork TEXT NOT NULL,
+    SocialAction TEXT NOT NULL,
+    HTTPError SMALLINT NOT NULL,
+    SendTiming INTEGER NOT NULL,
+    DNSTiming INTEGER NOT NULL,
+    ConnectTiming INTEGER NOT NULL,
+    ResponseStartTiming INTEGER NOT NULL,
+    ResponseEndTiming INTEGER NOT NULL,
+    FetchTiming INTEGER NOT NULL,
+    SocialSourceNetworkID SMALLINT NOT NULL,
+    SocialSourcePage TEXT NOT NULL,
+    ParamPrice BIGINT NOT NULL,
+    ParamOrderID TEXT NOT NULL,
+    ParamCurrency TEXT NOT NULL,
+    ParamCurrencyID SMALLINT NOT NULL,
+    OpenstatServiceName TEXT NOT NULL,
+    OpenstatCampaignID TEXT NOT NULL,
+    OpenstatAdID TEXT NOT NULL,
+    OpenstatSourceID TEXT NOT NULL,
+    UTMSource TEXT NOT NULL,
+    UTMMedium TEXT NOT NULL,
+    UTMCampaign TEXT NOT NULL,
+    UTMContent TEXT NOT NULL,
+    UTMTerm TEXT NOT NULL,
+    FromTag TEXT NOT NULL,
+    HasGCLID SMALLINT NOT NULL,
+    RefererHash BIGINT NOT NULL,
+    URLHash BIGINT NOT NULL,
+    CLID INTEGER NOT NULL,
+    PRIMARY KEY (CounterID, EventDate, UserID, EventTime, WatchID)
+)USING columnar;
+

tembo-olap/index.sql

@@ -0,0 +1,18 @@
+CREATE INDEX adveng on hits (AdvEngineID);
+CREATE INDEX regid  on hits (RegionID);
+CREATE INDEX cid on hits (CounterID);
+CREATE INDEX eventtime on hits (EventTime);
+CREATE INDEX eventdate on hits (EventDate);
+CREATE INDEX mobile on hits (MobilePhoneModel);
+CREATE INDEX refresh on hits (IsRefresh, DontCountHits);
+CREATE INDEX resolutionwidth on hits (ResolutionWidth);
+CREATE INDEX search on hits (SearchPhrase);
+CREATE INDEX UserID on hits (UserID);
+
+CREATE INDEX UserIDsearch on hits (UserID, SearchPhrase);
+CREATE INDEX widcip on hits (WatchID, ClientIP);
+CREATE INDEX mobileuser on hits (MobilePhoneModel,UserID);
+CREATE INDEX regionuser on hits (RegionID,UserID);
+
+CREATE INDEX mobile2 on hits (MobilePhoneModel) WHERE MobilePhoneModel <> ''::text;
+CREATE INDEX search2 on hits (SearchPhrase) WHERE SearchPhrase <> ''::text;