Merge pull request #162 from paradedb/main

Author: nickitat

index.html

@@ -405,6 +405,7 @@
 ,{"system":"MongoDB","date":"2022-10-25","machine":"c6a.4xlarge, 500gb gp2","cluster_size":1,"comment":"","tags":["row-oriented","document","C++"],"load_time":44824,"data_size":86397751448,"result":[[64.713,63.126,62.48],[0.402,0.398,0.398],[116.556,115.775,115.935],[116.883,92.054,91.646],[28.523,25.694,25.618],[19.974,10.235,10.204],[0.005,0.003,0.003],[0.656,0.644,0.65],[390.984,377.012,377.204],[2331.229,2617.011,2659.976],[403.396,410.754,389.201],[400.136,391.611,384.638],[90.255,70.314,70.648],[171.232,133.632,133.416],[57.835,57.8,57.136],[474.638,421.62,419.999],[332.07,266.801,266.967],[246.706,246.484,246.741],[466.25,465.911,465.399],[0.01,0.003,0.002],[238.439,70.732,69.911],[152.131,20.13,20.276],[28.018,23.201,23.292],[118.896,79.388,74.994],[21.888,11.393,11.423],[0.005,0.002,0.002],[31.821,15.74,15.353],[113.3,110.56,109.007],[287.151,178.123,177.878],[1099.097,1076.143,1064.104],[4987.582,4947.372,4873.488],[4816.666,4748.195,4823.137],[982.576,933.784,912.187],[1081.243,706.058,585.97],[593.68,570.279,594.089],[544.094,427.503,419.334],[11.766,5.643,4.975],[1.628,1.651,1.626],[3.444,2.222,2.232],[10.562,8.762,9.151],[0.724,0.445,0.446],[0.423,0.413,0.388],[1.601,2.202,2.375]],"source":"mongodb/results/c6a.4xlarge.json"}
 ,{"system":"MySQL (MyISAM)","date":"2022-07-01","machine":"c6a.4xlarge, 500gb gp2","cluster_size":1,"comment":"","tags":["C++","row-oriented","MySQL compatible"],"load_time":2512,"data_size":121588958061,"result":[[0,0,0],[283.32,276.83,274.52],[276.93,278.29,283.27],[28.83,23.63,21.55],[46.41,40.81,40.93],[467.04,467.39,469.08],[31.02,25.89,24.2],[277.89,275.3,277.3],[329.34,325.8,325.35],[342.86,338.43,336.95],[282.03,279.87,281.22],[277.74,282.68,282],[335.66,334.83,336.44],[305.24,310.39,307.3],[337.41,338.52,342.94],[308.66,307.34,306.27],[738.38,748.44,740.75],[738.75,734.01,738.25],[867.01,872.92,868.84],[25.65,20.61,18.46],[312.39,313.67,306.66],[301.66,305.12,308.01],[298.12,298.44,312.4],[311.34,309.9,311.85],[281.87,278.5,275],[277.46,277.46,277.46],[280.75,278.04,281.76],[263.9,417.39,406.88],[707.21,711.96,705],[668.1,668.33,665.96],[330.31,333.36,331.94],[506.57,506.18,500.53],[2604.49,2681.96,2703.12],[830.65,832.88,831.14],[831.98,830.46,833.41],[608.49,608.51,613.68],[4.56,4.13,4.16],[3.8,3.8,3.7],[1.65,1.45,1.46],[6.33,5.14,6.15],[1.6,1.41,1.41],[1.56,1.42,1.39],[7.04,1.17,1.13]],"source":"mysql-myisam/results/c6a.4xlarge.json"}
 ,{"system":"MySQL","date":"2022-07-01","machine":"c6a.4xlarge, 500gb gp2","cluster_size":1,"comment":"","tags":["C++","row-oriented","MySQL compatible"],"load_time":9472,"data_size":171953585825,"result":[[339.77,339.88,339.77],[364.91,371.86,367.55],[366.2,368.91,389.66],[364.39,377.53,571.45],[377.69,390.02,384.86],[569.48,576.51,574.68],[367.4,368.23,370.41],[371.29,384.02,613.22],[478.85,683.22,495.68],[489.9,635.96,662.43],[386.07,396.49,640.15],[389.13,412.55,444.12],[447.97,455.54,448.06],[423.22,845.44,813.6],[452.48,460.07,453.98],[577.54,623.21,586.49],[852.07,856.36,862.66],[838.09,848.92,851.12],[1006.37,1011.16,1023.17],[369.76,375.61,415.28],[412.45,419.9,456.62],[411.65,432.88,482.2],[412.73,420.73,429.5],[551.16,577.62,545.45],[382.89,394.76,386.37],[380.9,391.4,385.05],[385.3,394.67,460.32],[388.95,394.7,387.21],[800.33,807.9,807.11],[706.03,745.27,718.9],[450.9,489.59,530.97],[625.5,651.93,647.32],[2721.13,2792.12,2819.26],[945.9,954.94,957.54],[945.42,953.78,965.16],[684.36,716.29,708.75],[10.01,3.79,3.77],[7.48,3.32,3.27],[5.09,0.98,0.96],[8.7,4.77,4.68],[4.82,0.76,0.74],[4.46,0.77,0.75],[7.04,1.17,1.13]],"source":"mysql/results/c6a.4xlarge.json"}
+,{"hide":false,"system":"ParadeDB","date":"2024-01-22","machine":"c6a.4xlarge, 500gb gp2","cluster_size":1,"comment":"","tags":["Rust","row-oriented","column-oriented","search","PostgreSQL compatible"],"load_time":1268,"data_size":15415061091,"result":[[0.163905,0.021898,0.022008],[0.207702,0.09921,0.100067],[0.237625,0.118278,0.118759],[0.442946,0.126447,0.131097],[1.0277,0.904216,0.874925],[1.5306,1.33361,1.35094],[0.136153,0.050723,0.048972],[0.196955,0.097955,0.100465],[1.75463,1.62648,1.6515],[3.27018,3.18922,3.20045],[0.533817,0.369565,0.379801],[0.682929,0.409102,0.419692],[1.54178,1.33929,1.35776],[3.61625,2.76572,2.70619],[1.64899,1.44228,1.44821],[1.17511,1.00967,1.06343],[3.26609,2.8722,2.982],[3.10603,2.75286,2.82727],[7.32941,6.11626,6.0983],[0.544845,0.140834,0.139874],[9.70331,1.6328,1.62837],[11.4209,1.88226,1.84449],[22.8747,4.23374,4.25072],[58.9316,12.5984,12.3467],[3.15828,0.640643,0.636879],[0.835229,0.576707,0.569113],[3.07483,0.725442,0.695266],[9.77006,2.25871,2.28875],[9.48875,5.25401,5.2664],[0.758836,0.596066,0.604683],[2.42823,1.22871,1.22743],[5.7783,1.6789,1.66446],[9.08455,9.12525,9.37842],[12.2218,6.75614,6.71551],[12.6965,7.37193,7.32369],[2.14621,1.95328,1.90366],[0.156902,0.133997,0.134037],[0.22542,0.133992,0.133607],[0.216827,0.134193,0.133826],[0.18494,0.092106,0.092236],[0.217797,0.134427,0.135062],[0.221416,0.135605,0.134971],[0.264453,0.113144,0.113175]],"source":"paradedb/results/c6a.4xlarge.json"}
 ,{"system":"Pinot","date":"2022-07-01","machine":"c6a.4xlarge, 500gb gp2","cluster_size":1,"comment":"It successfully loaded only 94465149 out of 99997497 records. Some queries returned NullPointerException. The loading process is painful - splitting to 100 pieces required. It does not correctly report errors on data loading, the results may be incorrect.","tags":["Java","column-oriented"],"load_time":2032,"data_size":null,"result":[[0.002,0.001,0.001],[0.186,0.186,0.185],[0.251,0.276,0.258],[0.475,0.281,0.238],[3.907,3.655,3.633],[30.471,14.687,14.93],[null,null,null],[0.135,0.134,0.148],[3.039,2.902,2.938],[3.159,3.212,3.225],[4.217,4.197,4.384],[4.145,4.124,4.121],[2.989,3.145,3.18],[6.402,6.886,6.374],[3.245,3.35,3.129],[5.112,5.027,5.141],[5.509,5.279,5.257],[0.865,0.856,0.829],[null,null,null],[0.017,0.015,0.015],[54.348,19.562,19.128],[null,null,null],[76.596,74.719,14.228],[7.441,5.77,5.87],[0.376,0.327,0.286],[7.689,0.395,1.281],[3.434,0.499,0.5],[27.679,2.378,2.393],[null,null,null],[2.221,2.227,2.167],[4.941,4.639,4.565],[5.641,5.37,5.007],[5.295,5.006,5.357],[5.28,5.21,5.105],[6.231,6.238,6.385],[5.918,5.933,5.934],[0.26,0.202,0.21],[0.364,0.072,0.069],[0.042,0.034,0.035],[1.483,0.686,0.651],[0.113,0.071,0.079],[0.042,0.051,0.037],[null,null,null]],"source":"pinot/results/c6a.4xlarge.json"}
 ,{"hide":true,"system":"PostgreSQL (tuned)","date":"2022-12-18","machine":"c6a.4xlarge, 500gb gp2","cluster_size":1,"comment":"","tags":["C","row-oriented","PostgreSQL compatible"],"load_time":11800,"data_size":128875248605,"result":[[4.10957,0.687829,0.702019],[3.92694,0.63874,0.636411],[262.331,262.793,262.826],[6.14366,1.16457,1.20297],[26.752,27.4474,27.4878],[50.6555,47.6136,49.2373],[0.035605,0.002266,0.002208],[1.07557,0.599462,0.597673],[18.537,17.4348,17.7942],[309.49,306.831,306.894],[7.78261,2.95749,3.00855],[325.58,315.688,319.237],[4.03189,2.31557,2.21055],[18.8998,10.0497,10.0307],[272.381,271.91,271.87],[5.91906,4.17788,4.21394],[15.6578,15.8794,15.937],[0.025629,0.001484,0.00153],[312.64,312.543,310.582],[0.026206,0.001114,0.001115],[13.1447,0.090463,0.104537],[0.11893,0.093162,0.090471],[18.7775,0.107025,0.110361],[0.131412,0.089608,0.087852],[0.048958,0.00153,0.001405],[0.016134,0.001351,0.001268],[0.021939,0.001781,0.00164],[265.557,260.971,261.025],[277.516,276.053,277.378],[7.56565,6.50964,6.54902],[266.723,266.389,266.231],[269.738,269.255,269.193],[331.115,330.011,330.913],[416.169,421.255,423.724],[408.434,409.538,407.552],[35.5578,18.1707,18.6893],[3.17403,0.652905,0.698142],[0.730699,0.489315,0.465665],[0.563463,0.319131,0.3182],[1.23744,0.989391,1.01963],[0.752814,0.509212,0.511453],[0.677541,0.435178,0.437736],[1.18281,0.688545,0.697523]],"source":"postgresql-tuned/results/c6a.4xlarge.json"}
 ,{"system":"PostgreSQL","date":"2022-07-01","machine":"c6a.4xlarge, 500gb gp2","cluster_size":1,"comment":"","tags":["C","row-oriented","PostgreSQL compatible"],"load_time":2342,"data_size":77797067741,"result":[[439.753,309.785,282.017],[317.874,254.238,254.941],[262.883,263.072,263.09],[32.421,5.31,5.06],[57.134,42.648,42.334],[358.423,356.315,358.342],[31.524,5.35,4.994],[263.145,263.193,263.165],[323.659,322.858,321.918],[327.395,326.17,326.231],[265.983,265.681,265.912],[269.984,265.336,265.379],[284.096,284.56,282.234],[277.25,279.455,280.035],[285.66,286.2,283.611],[66.605,32.023,38.282],[312.452,304.431,305.391],[289.209,290.449,287.578],[331.706,327.485,334.428],[24.646,2.543,2.263],[267.561,267.496,267.524],[267.729,267.69,268.184],[263.074,263.12,267.04],[267.602,267.488,267.494],[263.141,263.859,263.137],[262.923,263.102,263.113],[262.885,263.088,263.114],[267.864,269.127,268.204],[303.376,306.925,308.664],[263.221,263.119,263.148],[270.814,270.575,270.294],[278.342,275.925,276.224],[584.599,576.932,591.502],[462.576,446.962,439.779],[429.93,417.696,416.704],[296.875,297.283,295.14],[3.461,0.842,0.794],[2.179,0.564,0.558],[2.258,0.566,0.416],[2.805,1.311,1.317],[2.936,0.82,0.615],[2.197,0.736,0.535],[1.983,0.32,0.312]],"source":"postgresql/results/c6a.4xlarge.json"}

paradedb/README.md

@@ -0,0 +1,23 @@
+# ParadeDB
+
+ParadeDB is an alternative to Elasticsearch built on Postgres.
+
+- [GitHub](https://github.com/paradedb/paradedb)
+- [Homepage](https://paradedb.com)
+
+The published benchmarks are based on ParadeDB version `v0.5.1`.
+
+## Benchmarks
+
+To run the benchmarks yourself:
+
+1. Manually start an AWS EC2 instance
+   - `c6a.4xlarge`
+   - Ubuntu Server 22.04 LTS (HVM), SSD Volume Type
+   - Root 500GB gp2 SSD
+2. Wait for the status check to pass, then SSH into the instance via EC2 Instance Connect
+3. Clone this repository via `git clone https://github.com/ClickHouse/ClickBench`
+4. Navigate to the `paradedb` directory via `cd ClickBench/paradedb`
+5. Run the benchmark via `./benchmark.sh`
+
+The benchmark should be completed in under an hour. If you'd like to benchmark against a different version of ParadeDB, modify the Docker tag in `benchmark.sh`. You can find the list of available tags [here](https://hub.docker.com/r/paradedb/paradedb/tags).

paradedb/benchmark.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+# Cleanup function to reset the environment
+cleanup() {
+    echo ""
+    echo "Cleaning up..."
+    if sudo docker ps -q --filter "name=paradedb" | grep -q .; then
+        sudo docker kill paradedb
+    fi
+    sudo docker rm paradedb
+    echo "Done, goodbye!"
+}
+
+# Register the cleanup function to run when the script exits
+trap cleanup EXIT
+
+sudo apt-get update
+sudo apt-get install -y docker.io
+sudo apt-get install -y postgresql-client
+
+if [ ! -e hits.tsv ]; then
+    echo ""
+    echo "Downloading dataset..."
+    wget --no-verbose --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+    gzip -d hits.tsv.gz
+else
+    echo ""
+    echo "Dataset already downloaded, skipping..."
+fi
+
+echo ""
+echo "Pulling ParadeDB image..."
+sudo docker run \
+    -e POSTGRES_USER=myuser \
+    -e POSTGRES_PASSWORD=mypassword \
+    -e POSTGRES_DB=mydb \
+    -p 5432:5432 \
+    --name paradedb \
+    -d \
+    paradedb/paradedb:0.5.1
+
+echo ""
+echo "Waiting for ParadeDB to start..."
+sleep 10
+
+echo ""
+echo "Loading dataset..."
+export PGPASSWORD='mypassword'
+psql -h localhost -U myuser -d mydb -p 5432 -t < create.sql
+psql -h localhost -U myuser -d mydb -p 5432 -t -c 'CALL paradedb.init();' -c '\timing' -c "\\copy hits FROM 'hits.tsv'"
+
+# COPY 99997497
+# Time: 1268695.244 ms (21:08.695)
+
+echo ""
+echo "Running queries..."
+./run.sh 2>&1 | tee log.txt
+
+sudo docker exec -it paradedb du -bcs /var/lib/postgresql/data
+
+# 15415061091     /var/lib/postgresql/data
+# 15415061091     total
+
+cat log.txt | grep -oP 'Time: \d+\.\d+ ms' | sed -r -e 's/Time: ([0-9]+\.[0-9]+) ms/\1/' |
+    awk '{ if (i % 3 == 0) { printf "[" }; printf $1 / 1000; if (i % 3 != 2) { printf "," } else { print "]," }; ++i; }'

paradedb/create.sql

@@ -0,0 +1,109 @@
+CALL paradedb.init();
+CREATE TABLE IF NOT EXISTS hits
+(
+    WatchID BIGINT NOT NULL,
+    JavaEnable SMALLINT NOT NULL,
+    Title TEXT NOT NULL,
+    GoodEvent SMALLINT NOT NULL,
+    EventTime TIMESTAMP NOT NULL,
+    EventDate Date NOT NULL,
+    CounterID INTEGER NOT NULL,
+    ClientIP INTEGER NOT NULL,
+    RegionID INTEGER NOT NULL,
+    UserID BIGINT NOT NULL,
+    CounterClass SMALLINT NOT NULL,
+    OS SMALLINT NOT NULL,
+    UserAgent SMALLINT NOT NULL,
+    URL TEXT NOT NULL,
+    Referer TEXT NOT NULL,
+    IsRefresh SMALLINT NOT NULL,
+    RefererCategoryID SMALLINT NOT NULL,
+    RefererRegionID INTEGER NOT NULL,
+    URLCategoryID SMALLINT NOT NULL,
+    URLRegionID INTEGER NOT NULL,
+    ResolutionWidth SMALLINT NOT NULL,
+    ResolutionHeight SMALLINT NOT NULL,
+    ResolutionDepth SMALLINT NOT NULL,
+    FlashMajor SMALLINT NOT NULL,
+    FlashMinor SMALLINT NOT NULL,
+    FlashMinor2 TEXT NOT NULL,
+    NetMajor SMALLINT NOT NULL,
+    NetMinor SMALLINT NOT NULL,
+    UserAgentMajor SMALLINT NOT NULL,
+    UserAgentMinor VARCHAR(255) NOT NULL,
+    CookieEnable SMALLINT NOT NULL,
+    JavascriptEnable SMALLINT NOT NULL,
+    IsMobile SMALLINT NOT NULL,
+    MobilePhone SMALLINT NOT NULL,
+    MobilePhoneModel TEXT NOT NULL,
+    Params TEXT NOT NULL,
+    IPNetworkID INTEGER NOT NULL,
+    TraficSourceID SMALLINT NOT NULL,
+    SearchEngineID SMALLINT NOT NULL,
+    SearchPhrase TEXT NOT NULL,
+    AdvEngineID SMALLINT NOT NULL,
+    IsArtifical SMALLINT NOT NULL,
+    WindowClientWidth SMALLINT NOT NULL,
+    WindowClientHeight SMALLINT NOT NULL,
+    ClientTimeZone SMALLINT NOT NULL,
+    ClientEventTime TIMESTAMP NOT NULL,
+    SilverlightVersion1 SMALLINT NOT NULL,
+    SilverlightVersion2 SMALLINT NOT NULL,
+    SilverlightVersion3 INTEGER NOT NULL,
+    SilverlightVersion4 SMALLINT NOT NULL,
+    PageCharset TEXT NOT NULL,
+    CodeVersion INTEGER NOT NULL,
+    IsLink SMALLINT NOT NULL,
+    IsDownload SMALLINT NOT NULL,
+    IsNotBounce SMALLINT NOT NULL,
+    FUniqID BIGINT NOT NULL,
+    OriginalURL TEXT NOT NULL,
+    HID INTEGER NOT NULL,
+    IsOldCounter SMALLINT NOT NULL,
+    IsEvent SMALLINT NOT NULL,
+    IsParameter SMALLINT NOT NULL,
+    DontCountHits SMALLINT NOT NULL,
+    WithHash SMALLINT NOT NULL,
+    HitColor CHAR NOT NULL,
+    LocalEventTime TIMESTAMP NOT NULL,
+    Age SMALLINT NOT NULL,
+    Sex SMALLINT NOT NULL,
+    Income SMALLINT NOT NULL,
+    Interests SMALLINT NOT NULL,
+    Robotness SMALLINT NOT NULL,
+    RemoteIP INTEGER NOT NULL,
+    WindowName INTEGER NOT NULL,
+    OpenerName INTEGER NOT NULL,
+    HistoryLength SMALLINT NOT NULL,
+    BrowserLanguage TEXT NOT NULL,
+    BrowserCountry TEXT NOT NULL,
+    SocialNetwork TEXT NOT NULL,
+    SocialAction TEXT NOT NULL,
+    HTTPError SMALLINT NOT NULL,
+    SendTiming INTEGER NOT NULL,
+    DNSTiming INTEGER NOT NULL,
+    ConnectTiming INTEGER NOT NULL,
+    ResponseStartTiming INTEGER NOT NULL,
+    ResponseEndTiming INTEGER NOT NULL,
+    FetchTiming INTEGER NOT NULL,
+    SocialSourceNetworkID SMALLINT NOT NULL,
+    SocialSourcePage TEXT NOT NULL,
+    ParamPrice BIGINT NOT NULL,
+    ParamOrderID TEXT NOT NULL,
+    ParamCurrency TEXT NOT NULL,
+    ParamCurrencyID SMALLINT NOT NULL,
+    OpenstatServiceName TEXT NOT NULL,
+    OpenstatCampaignID TEXT NOT NULL,
+    OpenstatAdID TEXT NOT NULL,
+    OpenstatSourceID TEXT NOT NULL,
+    UTMSource TEXT NOT NULL,
+    UTMMedium TEXT NOT NULL,
+    UTMCampaign TEXT NOT NULL,
+    UTMContent TEXT NOT NULL,
+    UTMTerm TEXT NOT NULL,
+    FromTag TEXT NOT NULL,
+    HasGCLID SMALLINT NOT NULL,
+    RefererHash BIGINT NOT NULL,
+    URLHash BIGINT NOT NULL,
+    CLID INTEGER NOT NULL
+) USING deltalake;