Merge branch 'main' into main

Author: osawyerr

index.html

@@ -433,9 +433,9 @@
 ,{"system":"StarRocks","date":"2023-02-10","machine":"c6a.metal, 500gb gp2","cluster_size":1,"comment":"","tags":["C++","column-oriented","MySQL compatible","ClickHouse derivative"],"load_time":433,"data_size":17705230744,"result":[[0.04,0.04,0.03],[0.12,0.03,0.03],[0.66,0.06,0.06],[2.08,0.08,0.06],[0.10,0.08,0.09],[2.19,0.22,0.21],[0.04,0.02,0.03],[0.06,0.04,0.04],[0.99,0.12,0.12],[0.73,0.74,0.70],[0.13,0.06,0.05],[1.04,0.06,0.07],[0.17,0.18,0.17],[0.23,0.23,0.23],[0.54,0.18,0.18],[0.09,0.09,0.08],[0.31,0.32,0.30],[0.12,0.10,0.11],[1.27,0.48,0.44],[0.01,0.01,0.00],[12.09,0.17,0.15],[0.17,0.13,0.12],[10.91,0.28,0.24],[28.25,0.86,1.36],[0.03,0.03,0.02],[0.06,0.04,0.04],[0.02,0.03,0.02],[0.63,0.24,0.21],[8.77,1.60,1.64],[0.12,0.10,0.10],[1.33,0.13,0.13],[3.46,0.18,0.19],[0.97,0.86,0.86],[0.95,0.96,0.98],[0.96,0.97,0.97],[0.14,0.12,0.14],[0.07,0.06,0.06],[0.05,0.04,0.04],[0.04,0.03,0.03],[0.10,0.09,0.09],[0.47,0.03,0.03],[0.25,0.03,0.03],[0.04,0.03,0.03]],"source":"starrocks/results/c6a.metal.json"}
 ,{"hide":false,"system":"Tablespace","date":"2024-02-11","machine":"L1 - 16CPU 32GB","cluster_size":1,"comment":"","tags":["C","Rust","row-oriented","column-oriented","PostgreSQL compatible","managed","analytical"],"load_time":14891,"data_size":122870918103,"result":[[0.307199,0.02727,0.027759],[0.374836,0.059755,0.058116],[0.20127,0.095664,0.097262],[1.01148,0.092086,0.091909],[0.884588,0.798001,0.781532],[2.74339,1.27036,1.24833],[0.008657,0.002024,0.002075],[0.106097,0.061991,0.073671],[1.19617,1.312,1.28469],[1.3998,1.35227,1.39408],[0.347245,0.325939,0.319456],[0.366212,0.343858,0.363897],[1.16179,1.15186,1.26195],[1.93425,1.90117,1.92537],[1.37206,1.30908,1.27425],[1.00939,1.04474,0.975766],[2.24397,2.231,2.33471],[0.01139,0.002308,0.002349],[4.58354,4.12992,4.11501],[0.006284,0.001806,0.001817],[14.4282,0.172332,0.165752],[0.17184,0.172279,0.170168],[20.2489,0.243308,0.233888],[0.169122,0.16989,0.164747],[0.042381,0.002397,0.002346],[0.006727,0.002257,0.002312],[0.013321,0.00274,0.002751],[21.7536,1.43192,1.46996],[16.5725,4.27724,4.29998],[0.599498,0.584713,0.537117],[1.1458,1.1857,1.1733],[3.93642,1.29765,1.37356],[6.89482,10.6813,5.74769],[46.6404,4.72151,4.45393],[4.88564,4.91751,4.79894],[1.91983,1.8092,1.85409],[0.424691,0.350219,0.324188],[0.649774,0.484059,0.483278],[0.178715,0.177479,0.1585],[0.832879,0.778821,0.800005],[0.145597,0.128932,0.085888],[0.088149,0.082234,0.082566],[0.115408,0.097142,0.096599]],"source":"tablespace/results/large-1.json"}
 ,{"system":"TimescaleDB (compression)","date":"2022-07-01","machine":"c6a.4xlarge, 500gb gp2","cluster_size":1,"comment":"","tags":["C","PostgreSQL compatible","column-oriented","time-series"],"load_time":4605,"data_size":20333747165,"result":[[2.28686,1.63642,1.64263],[32.6848,1.63476,1.40052],[60.8633,3.70484,3.59342],[36.4029,2.87091,2.80739],[110.391,38.9688,38.0549],[147.379,66.2513,65.6379],[33.0294,2.92031,2.84375],[33.0221,1.2984,1.19227],[115.694,47.4651,47.0125],[156.347,51.577,51.2694],[68.3301,4.75521,4.68007],[77.4356,5.55128,5.56577],[49.7741,11.2911,11.3265],[81.1014,14.9111,14.9541],[82.9569,14.6156,14.6331],[62.0338,26.399,26.3351],[103.259,36.4122,36.6076],[92.8828,26.2395,25.8991],[144.281,63.5102,63.7661],[7.00679,0.573073,0.536283],[75.0203,7.86344,7.90495],[81.2825,9.15868,9.01775],[104.084,13.9528,13.8435],[132.531,81.522,82.1561],[80.6965,3.28231,3.16574],[39.7693,2.51443,2.43849],[80.4245,3.26941,3.13916],[104.015,13.7044,13.5313],[307.26,253.127,252.147],[42.8549,22.4187,22.0325],[137.601,14.9592,14.6804],[136.767,22.8007,22.131],[263.005,168.551,163.355],[156.919,92.6308,91.702],[160.842,96.0512,97.1773],[62.8357,28.0336,28.7397],[1.75869,0.561604,0.541215],[0.46607,0.191863,0.19021],[0.303671,0.137579,0.136615],[2.32031,1.49223,1.52369],[0.563764,0.14192,0.138234],[0.372428,0.122989,0.123709],[0.448574,0.159092,0.154687]],"source":"timescaledb-compressed/results/c6a.4xlarge.json"}
-,{"system":"TimescaleDB","date":"2022-07-01","machine":"c6a.4xlarge, 500gb gp2","cluster_size":1,"comment":"","tags":["C","PostgreSQL compatible","row-oriented","time-series"],"load_time":1620,"data_size":72882392030,"result":[[437.700,215.793,176.420],[327.026,259.568,244.578],[262.978,263.090,263.083],[262.807,263.046,266.847],[337.497,334.964,330.852],[355.689,356.801,362.894],[262.762,263.012,262.968],[263.055,263.016,263.028],[319.928,319.388,320.704],[323.584,322.224,322.488],[265.979,265.465,265.375],[266.019,265.543,265.462],[277.018,276.300,276.595],[280.352,279.251,279.572],[279.915,279.896,279.674],[296.377,298.506,297.659],[314.448,314.605,312.570],[302.668,302.672,303.039],[325.810,324.061,324.376],[262.447,262.698,262.704],[267.581,267.467,267.482],[268.085,267.466,267.696],[263.391,263.097,263.126],[38.291,0.435,0.335],[0.127,0.005,0.005],[263.138,263.100,263.092],[0.889,0.341,0.339],[267.586,267.498,267.491],[289.086,290.012,290.093],[263.220,263.071,263.109],[274.780,273.995,273.998],[282.217,281.390,281.470],[429.273,426.588,439.431],[448.808,418.724,418.207],[455.196,422.750,423.142],[299.263,296.937,297.261],[18.693,1.552,1.481],[18.125,0.945,0.937],[18.528,1.062,0.902],[18.268,1.779,1.770],[19.615,1.965,1.966],[18.970,1.435,1.430],[18.330,1.153,0.952]],"source":"timescaledb/results/c6a.4xlarge.json"}
-,{"system":"Umbra","date":"2024-02-02","machine":"c6a.4xlarge, 500gb gp2","cluster_size":1,"tags":["C++","column-oriented","PostgreSQL compatible"],"load_time":1061.541,"data_size":97767877183,"comment":"Segmentation fault in the middle of the test","result":[[0.209086,0.011738,0.012061],[0.286187,0.011852,0.003785],[0.295367,0.135114,0.044054],[0.895806,0.679204,0.680544],[0.702135,0.674911,0.677621],[4.0023,0.87053,0.613472],[0.11274,0.078905,0.087947],[0.012837,0.014259,0.014031],[2.729,0.350596,0.341228],[0.696983,0.639588,0.669598],[0.339289,0.038563,0.043669],[0.209976,0.05442,0.054552],[0.430673,0.587625,0.768441],[0.932235,0.787055,0.780889],[1.28999,0.276156,0.297337],[0.202715,0.188756,0.192702],[0.530036,0.623375,0.608413],[0.511773,0.595728,0.563421],[1.42062,1.23585,1.192],[0.035434,0.013534,0.002076],[27.6449,14.838,12.4979],[11.6779,6.98523,4.57714],[28.1569,20.2952,10.3134],[153.316,153.2,154.629],[4.08712,1.57023,0.007798],[0.010185,0.010279,0.009956],[0.038879,0.038326,0.039015],[19.3634,7.89121,4.27973],[16.7256,10.2004,3.92077],[0.303641,0.054993,0.055398],[4.58902,0.332033,0.130684],[2.79384,1.48084,0.731322],[1.38139,1.6159,1.39411],[12.9291,9.05383,6.94245],[5.3304,5.41123,5.38231],[0.57384,0.254843,0.26153],[null,null,null],[null,null,null],[null,null,null],[null,null,null],[null,null,null],[null,null,null],[null,null,null]],"source":"umbra/results/c6a.4xlarge.json"}
-,{"system":"Umbra","date":"2024-02-02","machine":"c6a.metal, 500gb gp2","cluster_size":1,"tags":["C++","column-oriented","PostgreSQL compatible"],"load_time":432.753,"data_size":104811147008,"comment":"It looks like it caches the data in memory, so the cold run results are incorrect. When I attempt to restart the server, it didn't start due to Segmentation fault.","hide":true,"result":[[0.021103,0.014069,0.013841],[0.016241,0.015651,0.014398],[0.016433,0.019444,0.020535],[0.019118,0.022744,0.024424],[0.087463,0.116452,0.108668],[0.137545,0.143623,0.186976],[0.022322,0.018086,0.016414],[0.01743,0.01459,0.016214],[0.13723,0.145615,0.154707],[0.602285,0.614764,0.614396],[0.052353,0.055475,0.060419],[0.066132,0.068412,0.064872],[0.143196,0.178901,0.184571],[0.246122,0.285504,0.287426],[0.141119,0.172389,0.192775],[0.133211,0.124609,0.127582],[0.187477,0.219748,0.214415],[0.157956,0.187116,0.206256],[0.267917,0.337226,0.368166],[0.036774,0.019947,0.019828],[0.180748,0.088995,0.092018],[0.050261,0.056437,0.049835],[0.085935,0.08502,0.085339],[0.384127,0.283856,0.298186],[0.023887,0.025708,0.029501],[0.026465,0.022693,0.026492],[0.038385,0.043431,0.041599],[0.108191,0.095912,0.094505],[0.384992,0.369839,0.397459],[0.023921,0.019091,0.020356],[0.105971,0.103385,0.108233],[0.133147,0.127664,0.13925],[0.332119,0.366251,0.336184],[0.592509,0.652576,0.750742],[0.839829,0.668914,0.731482],[0.386763,0.131991,0.143009],[0.038283,0.033977,0.03078],[0.02885,0.021694,0.021639],[0.025068,0.018045,0.0184],[0.041889,0.041119,0.041826],[0.030802,0.021935,0.029303],[0.020964,0.022451,0.023481],[0.017381,0.020976,0.016867]],"source":"umbra/results/c6a.metal.json"}
+,{"system":"TimescaleDB","date":"2022-07-01","machine":"c6a.4xlarge, 500gb gp2","cluster_size":1,"comment":"","tags":["C","PostgreSQL compatible","row-oriented","time-series"],"load_time":1620,"data_size":72882392030,"result":[[437.7,215.793,176.42],[327.026,259.568,244.578],[262.978,263.09,263.083],[262.807,263.046,266.847],[337.497,334.964,330.852],[355.689,356.801,362.894],[262.762,263.012,262.968],[263.055,263.016,263.028],[319.928,319.388,320.704],[323.584,322.224,322.488],[265.979,265.465,265.375],[266.019,265.543,265.462],[277.018,276.3,276.595],[280.352,279.251,279.572],[279.915,279.896,279.674],[296.377,298.506,297.659],[314.448,314.605,312.57],[302.668,302.672,303.039],[325.81,324.061,324.376],[262.447,262.698,262.704],[267.581,267.467,267.482],[268.085,267.466,267.696],[263.391,263.097,263.126],[38.291,0.435,0.335],[0.127,0.005,0.005],[263.138,263.1,263.092],[0.889,0.341,0.339],[267.586,267.498,267.491],[289.086,290.012,290.093],[263.22,263.071,263.109],[274.78,273.995,273.998],[282.217,281.39,281.47],[429.273,426.588,439.431],[448.808,418.724,418.207],[455.196,422.75,423.142],[299.263,296.937,297.261],[18.693,1.552,1.481],[18.125,0.945,0.937],[18.528,1.062,0.902],[18.268,1.779,1.77],[19.615,1.965,1.966],[18.97,1.435,1.43],[18.33,1.153,0.952]],"source":"timescaledb/results/c6a.4xlarge.json"}
+,{"system":"Umbra","date":"2024-02-05","machine":"c6a.4xlarge, 500gb gp2","cluster_size":1,"tags":["C++","column-oriented","PostgreSQL compatible"],"load_time":505.913,"data_size":36477780830,"result":[[0.030601,0.010866,0.011101],[0.33904,0.003698,0.003548],[0.645752,0.027352,0.026303],[0.719249,0.026984,0.026267],[0.727638,0.13537,0.133211],[2.0782,0.252201,0.255137],[0.338405,0.027632,0.027376],[0.309864,0.004253,0.004158],[1.61685,0.243389,0.263255],[3.01982,0.51973,0.524606],[1.08709,0.033062,0.032059],[1.46887,0.037179,0.035897],[2.0821,0.216457,0.215829],[3.92123,0.391088,0.395779],[2.48072,0.233635,0.21876],[0.765295,0.18303,0.18472],[3.88406,0.518152,0.541263],[3.8342,0.459446,0.476508],[7.15095,0.868591,0.862418],[0.674124,0.001847,0.001711],[18.1118,0.276739,0.27634],[21.1296,0.085333,0.083386],[35.0997,0.143139,0.141073],[122.717,118.44,116.582],[5.06751,0.009433,0.009186],[2.00484,0.00967,0.009456],[5.07312,0.038766,0.038597],[18.5152,0.269536,0.271205],[13.2374,2.55058,2.54432],[0.465351,0.022607,0.021943],[4.74853,0.13671,0.133112],[7.54723,0.204112,0.20182],[5.16736,1.29808,1.30411],[18.7229,1.4329,1.42222],[18.6891,1.47666,1.50563],[0.706767,0.194858,0.196652],[19.6567,0.01349,0.013724],[12.869,0.008628,0.009058],[1.23511,0.019785,0.019251],[34.5888,0.030742,0.027173],[0.503624,0.015988,0.015614],[0.288441,0.016486,0.016809],[3.59948,0.005299,0.005082]],"source":"umbra/results/c6a.4xlarge.json"}
+,{"system":"Umbra","date":"2024-02-05","machine":"c6a.metal, 500gb gp2","cluster_size":1,"tags":["C++","column-oriented","PostgreSQL compatible"],"load_time":224.253,"data_size":37985071698,"result":[[0.664962,0.014879,0.013896],[0.86897,0.014264,0.014216],[1.39741,0.019753,0.019888],[1.76497,0.022482,0.022356],[1.80665,0.067752,0.081],[3.30264,0.133111,0.156861],[0.785534,0.0208,0.022284],[0.870046,0.015516,0.016097],[2.66753,0.14137,0.15998],[4.51368,0.689448,0.726916],[2.19478,0.053639,0.059179],[2.57461,0.043306,0.045997],[3.30665,0.164523,0.175388],[4.92596,0.234748,0.273746],[3.70171,0.147932,0.182008],[1.851,0.094632,0.113691],[4.93327,0.243824,0.270263],[4.87596,0.226228,0.210788],[8.02335,0.306983,0.333771],[1.7747,0.017516,0.018097],[19.9175,0.097303,0.081085],[22.3949,0.049008,0.049755],[37.544,0.080033,0.08298],[127.193,0.237656,0.249893],[5.70401,0.02041,0.02158],[2.93367,0.020734,0.021048],[6.12096,0.034546,0.038698],[19.8159,0.110082,0.107088],[14.2241,0.336072,0.36425],[0.663126,0.019808,0.019036],[5.58319,0.082806,0.083288],[8.2146,0.106136,0.093486],[5.46498,0.342316,0.345279],[19.5496,0.631508,0.862308],[19.0042,0.849141,0.840496],[0.303872,0.119839,0.087051],[20.4269,0.029673,0.027629],[14.2258,0.02278,0.019946],[7.9593,0.012017,0.013362],[35.6343,0.039568,0.037401],[2.49159,0.014061,0.013447],[2.00332,0.012786,0.011437],[3.64226,0.016922,0.017801]],"source":"umbra/results/c6a.metal.json"}
 ]; // end of data
 
 const additional_data_size_points = [
@@ -987,7 +987,7 @@ <h2>Detailed Comparison</h2>
             const ratio = curr_timing !== null ? (constant_time_add + curr_timing) / (constant_time_add + baseline_timing) : null;
 
             let td = document.createElement('td');
-            td.appendChild(document.createTextNode(curr_timing !== null ? `${curr_timing.toFixed(2)}s (×${ratio.toFixed(2)})` : '☠'));
+            td.appendChild(document.createTextNode(curr_timing !== null ? `${curr_timing.toFixed(3)}s (×${ratio.toFixed(2)})` : '☠'));
 
             colorize(td, ratio);
             tr.appendChild(td);

umbra/benchmark.sh

@@ -1,34 +1,32 @@
 #!/bin/bash
 
+# Ubuntu
 sudo apt-get update
 sudo apt-get install -y postgresql-client gzip
 
+# Amazon Linux
+# yum install nc postgresql15
+
+rm -rf hits.tsv
 wget --no-verbose --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
 gzip -d hits.tsv.gz
-chmod 777 ~ hits.tsv
+chmod 777 hits.tsv
 
 rm -rf umbra-24-01-03.tar.xz umbra
-wget --no-verbose --continue 'https://db.in.tum.de/~schmidt/umbra-24-01-03.tar.xz'
-tar -xf umbra-24-01-03.tar.xz
+wget --no-verbose --continue 'https://db.in.tum.de/~schmidt/umbra-2024-02-04.tar.xz'
+tar -xf umbra-2024-02-04.tar.xz
 
 rm -rf db
 mkdir db
 
 export USEDIRECTIO=1
 umbra/bin/sql -createdb db/umbra.db <<<"ALTER ROLE postgres WITH LOGIN SUPERUSER PASSWORD 'password';"
 
-umbra/bin/server -createSSLFiles -certFile db/umbra.cert -keyFile db/umbra.pem -address 0.0.0.0 db/umbra.db &> umbra.log &
-sleep 1
-
-psql -h /tmp -U postgres -t < create.sql
-psql -h /tmp -U postgres -t -c '\timing' -c "copy hits from 'hits.tsv' with (format text);"
+time umbra/bin/sql db/umbra.db create.sql
 
 ./run.sh 2>&1 | tee log.txt
 
 du -bcs db/
 
 cat log.txt | grep -oP 'Time: \d+\.\d+ ms' | sed -r -e 's/Time: ([0-9]+\.[0-9]+) ms/\1/' |
     awk '{ if (i % 3 == 0) { printf "[" }; printf $1 / 1000; if (i % 3 != 2) { printf "," } else { print "]," }; ++i; }'
-
-killall server
-rm -rf hits.tsv

umbra/create.sql

@@ -107,3 +107,5 @@ create table hits (
     primary key (counterid, eventdate, userid, eventtime, watchid)
 ) with (storage=columnar);
 
+copy hits from 'hits.tsv' with (format text);
+

umbra/results/c6a.4xlarge.json

@@ -1,55 +1,54 @@
 {
     "system": "Umbra",
-    "date": "2024-02-02",
+    "date": "2024-02-05",
     "machine": "c6a.4xlarge, 500gb gp2",
     "cluster_size": 1,
     "tags": ["C++", "column-oriented", "PostgreSQL compatible"],
-    "load_time": 1061.541,
-    "data_size": 97767877183,
-    "comment": "Segmentation fault in the middle of the test",
+    "load_time": 505.913,
+    "data_size": 36477780830,
     "result": [
-[0.209086,0.011738,0.012061],
-[0.286187,0.011852,0.003785],
-[0.295367,0.135114,0.044054],
-[0.895806,0.679204,0.680544],
-[0.702135,0.674911,0.677621],
-[4.0023,0.87053,0.613472],
-[0.11274,0.078905,0.087947],
-[0.012837,0.014259,0.014031],
-[2.729,0.350596,0.341228],
-[0.696983,0.639588,0.669598],
-[0.339289,0.038563,0.043669],
-[0.209976,0.05442,0.054552],
-[0.430673,0.587625,0.768441],
-[0.932235,0.787055,0.780889],
-[1.28999,0.276156,0.297337],
-[0.202715,0.188756,0.192702],
-[0.530036,0.623375,0.608413],
-[0.511773,0.595728,0.563421],
-[1.42062,1.23585,1.192],
-[0.035434,0.013534,0.002076],
-[27.6449,14.838,12.4979],
-[11.6779,6.98523,4.57714],
-[28.1569,20.2952,10.3134],
-[153.316,153.2,154.629],
-[4.08712,1.57023,0.007798],
-[0.010185,0.010279,0.009956],
-[0.038879,0.038326,0.039015],
-[19.3634,7.89121,4.27973],
-[16.7256,10.2004,3.92077],
-[0.303641,0.054993,0.055398],
-[4.58902,0.332033,0.130684],
-[2.79384,1.48084,0.731322],
-[1.38139,1.6159,1.39411],
-[12.9291,9.05383,6.94245],
-[5.3304,5.41123,5.38231],
-[0.57384,0.254843,0.26153],
-[null,null,null],
-[null,null,null],
-[null,null,null],
-[null,null,null],
-[null,null,null],
-[null,null,null],
-[null,null,null]
+        [0.030601,0.010866,0.011101],
+        [0.33904,0.003698,0.003548],
+        [0.645752,0.027352,0.026303],
+        [0.719249,0.026984,0.026267],
+        [0.727638,0.13537,0.133211],
+        [2.0782,0.252201,0.255137],
+        [0.338405,0.027632,0.027376],
+        [0.309864,0.004253,0.004158],
+        [1.61685,0.243389,0.263255],
+        [3.01982,0.51973,0.524606],
+        [1.08709,0.033062,0.032059],
+        [1.46887,0.037179,0.035897],
+        [2.0821,0.216457,0.215829],
+        [3.92123,0.391088,0.395779],
+        [2.48072,0.233635,0.21876],
+        [0.765295,0.18303,0.18472],
+        [3.88406,0.518152,0.541263],
+        [3.8342,0.459446,0.476508],
+        [7.15095,0.868591,0.862418],
+        [0.674124,0.001847,0.001711],
+        [18.1118,0.276739,0.27634],
+        [21.1296,0.085333,0.083386],
+        [35.0997,0.143139,0.141073],
+        [122.717,118.44,116.582],
+        [5.06751,0.009433,0.009186],
+        [2.00484,0.00967,0.009456],
+        [5.07312,0.038766,0.038597],
+        [18.5152,0.269536,0.271205],
+        [13.2374,2.55058,2.54432],
+        [0.465351,0.022607,0.021943],
+        [4.74853,0.13671,0.133112],
+        [7.54723,0.204112,0.20182],
+        [5.16736,1.29808,1.30411],
+        [18.7229,1.4329,1.42222],
+        [18.6891,1.47666,1.50563],
+        [0.706767,0.194858,0.196652],
+        [19.6567,0.01349,0.013724],
+        [12.869,0.008628,0.009058],
+        [1.23511,0.019785,0.019251],
+        [34.5888,0.030742,0.027173],
+        [0.503624,0.015988,0.015614],
+        [0.288441,0.016486,0.016809],
+        [3.59948,0.005299,0.005082]
     ]
 }