Skip to content

Commit 516700e

Browse files
rschu1zerschu1ze
authored
Merge pull request ClickHouse#315 from caetanosauer/hyper
Hyper: Fix EventTime handling in Parquet
2 parents b4fb374 + 4d1c6cd commit 516700e

File tree

3 files changed

+96
-96
lines changed

3 files changed

+96
-96
lines changed

‎hyper-parquet/queries.sql‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ SELECT SUM("AdvEngineID"), COUNT(*), AVG("ResolutionWidth") FROM hits;
44
SELECT AVG("UserID") FROM hits;
55
SELECT COUNT(DISTINCT "UserID") FROM hits;
66
SELECT COUNT(DISTINCT "SearchPhrase") FROM hits;
7-
SELECT to_timestamp(MIN("EventDate")), to_timestamp(MAX("EventDate")) FROM hits;
7+
SELECT (date '1970-01-01' + MIN("EventDate")), (date '1970-01-01' + MAX("EventDate")) FROM hits;
88
SELECT "AdvEngineID", COUNT(*) FROM hits WHERE "AdvEngineID" <> 0 GROUP BY "AdvEngineID" ORDER BY COUNT(*) DESC;
99
SELECT "RegionID", COUNT(DISTINCT "UserID") AS u FROM hits GROUP BY "RegionID" ORDER BY u DESC LIMIT 10;
1010
SELECT "RegionID", SUM("AdvEngineID"), COUNT(*) AS c, AVG("ResolutionWidth"), COUNT(DISTINCT "UserID") FROM hits GROUP BY "RegionID" ORDER BY c DESC LIMIT 10;
@@ -34,10 +34,10 @@ SELECT "WatchID", "ClientIP", COUNT(*) AS c, SUM("IsRefresh"), AVG("ResolutionWi
3434
SELECT cast("URL" as text), COUNT(*) AS c FROM hits GROUP BY cast("URL" as text) ORDER BY c DESC LIMIT 10;
3535
SELECT 1, cast("URL" as text), COUNT(*) AS c FROM hits GROUP BY 1, cast("URL" as text) ORDER BY c DESC LIMIT 10;
3636
SELECT "ClientIP", "ClientIP" - 1, "ClientIP" - 2, "ClientIP" - 3, COUNT(*) AS c FROM hits GROUP BY "ClientIP", "ClientIP" - 1, "ClientIP" - 2, "ClientIP" - 3 ORDER BY c DESC LIMIT 10;
37-
SELECT "URL", COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate") >= '2013-07-01' AND to_timestamp("EventDate") <= '2013-07-31' AND "DontCountHits" = 0 AND "IsRefresh" = 0 AND cast("URL" as text) <> '' GROUP BY "URL" ORDER BY "PageViews" DESC LIMIT 10;
38-
SELECT "Title", COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate") >= '2013-07-01' AND to_timestamp("EventDate") <= '2013-07-31' AND "DontCountHits" = 0 AND "IsRefresh" = 0 AND "Title" <> '' GROUP BY "Title" ORDER BY "PageViews" DESC LIMIT 10;
39-
SELECT "URL", COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate") >= '2013-07-01' AND to_timestamp("EventDate") <= '2013-07-31' AND "IsRefresh" = 0 AND "IsLink" <> 0 AND "IsDownload" = 0 GROUP BY "URL" ORDER BY "PageViews" DESC LIMIT 10 OFFSET 1000;
40-
SELECT "TraficSourceID", "SearchEngineID", "AdvEngineID", CASE WHEN ("SearchEngineID" = 0 AND "AdvEngineID" = 0) THEN "Referer" ELSE '' END AS Src, "URL" AS Dst, COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate") >= '2013-07-01' AND to_timestamp("EventDate") <= '2013-07-31' AND "IsRefresh" = 0 GROUP BY "TraficSourceID", "SearchEngineID", "AdvEngineID", Src, Dst ORDER BY "PageViews" DESC LIMIT 10 OFFSET 1000;
41-
SELECT "URLHash", "EventDate", COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate") >= '2013-07-01' AND to_timestamp("EventDate") <= '2013-07-31' AND "IsRefresh" = 0 AND "TraficSourceID" IN (-1, 6) AND "RefererHash" = 3594120000172545465 GROUP BY "URLHash", "EventDate" ORDER BY "PageViews" DESC LIMIT 10 OFFSET 100;
42-
SELECT "WindowClientWidth", "WindowClientHeight", COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate") >= '2013-07-01' AND to_timestamp("EventDate") <= '2013-07-31' AND "IsRefresh" = 0 AND "DontCountHits" = 0 AND "URLHash" = 2868770270353813622 GROUP BY "WindowClientWidth", "WindowClientHeight" ORDER BY "PageViews" DESC LIMIT 10 OFFSET 10000;
43-
SELECT DATE_TRUNC('minute', to_timestamp("EventTime")) AS M, COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate") >= '2013-07-14' AND to_timestamp("EventDate") <= '2013-07-15' AND "IsRefresh" = 0 AND "DontCountHits" = 0 GROUP BY DATE_TRUNC('minute', to_timestamp("EventTime")) ORDER BY DATE_TRUNC('minute', to_timestamp("EventTime")) LIMIT 10 OFFSET 1000;
37+
SELECT "URL", COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND (date '1970-01-01' + "EventDate") >= '2013-07-01' AND (date '1970-01-01' + "EventDate") <= '2013-07-31' AND "DontCountHits" = 0 AND "IsRefresh" = 0 AND cast("URL" as text) <> '' GROUP BY "URL" ORDER BY "PageViews" DESC LIMIT 10;
38+
SELECT "Title", COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND (date '1970-01-01' + "EventDate") >= '2013-07-01' AND (date '1970-01-01' + "EventDate") <= '2013-07-31' AND "DontCountHits" = 0 AND "IsRefresh" = 0 AND "Title" <> '' GROUP BY "Title" ORDER BY "PageViews" DESC LIMIT 10;
39+
SELECT "URL", COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND (date '1970-01-01' + "EventDate") >= '2013-07-01' AND (date '1970-01-01' + "EventDate") <= '2013-07-31' AND "IsRefresh" = 0 AND "IsLink" <> 0 AND "IsDownload" = 0 GROUP BY "URL" ORDER BY "PageViews" DESC LIMIT 10 OFFSET 1000;
40+
SELECT "TraficSourceID", "SearchEngineID", "AdvEngineID", CASE WHEN ("SearchEngineID" = 0 AND "AdvEngineID" = 0) THEN "Referer" ELSE '' END AS Src, "URL" AS Dst, COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND (date '1970-01-01' + "EventDate") >= '2013-07-01' AND (date '1970-01-01' + "EventDate") <= '2013-07-31' AND "IsRefresh" = 0 GROUP BY "TraficSourceID", "SearchEngineID", "AdvEngineID", Src, Dst ORDER BY "PageViews" DESC LIMIT 10 OFFSET 1000;
41+
SELECT "URLHash", (date '1970-01-01' + "EventDate"), COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND (date '1970-01-01' + "EventDate") >= '2013-07-01' AND (date '1970-01-01' + "EventDate") <= '2013-07-31' AND "IsRefresh" = 0 AND "TraficSourceID" IN (-1, 6) AND "RefererHash" = 3594120000172545465 GROUP BY "URLHash", (date '1970-01-01' + "EventDate") ORDER BY "PageViews" DESC LIMIT 10 OFFSET 100;
42+
SELECT "WindowClientWidth", "WindowClientHeight", COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND (date '1970-01-01' + "EventDate") >= '2013-07-01' AND (date '1970-01-01' + "EventDate") <= '2013-07-31' AND "IsRefresh" = 0 AND "DontCountHits" = 0 AND "URLHash" = 2868770270353813622 GROUP BY "WindowClientWidth", "WindowClientHeight" ORDER BY "PageViews" DESC LIMIT 10 OFFSET 10000;
43+
SELECT DATE_TRUNC('minute', to_timestamp("EventTime")) AS M, COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND (date '1970-01-01' + "EventDate") >= '2013-07-14' AND (date '1970-01-01' + "EventDate") <= '2013-07-15' AND "IsRefresh" = 0 AND "DontCountHits" = 0 GROUP BY DATE_TRUNC('minute', to_timestamp("EventTime")) ORDER BY DATE_TRUNC('minute', to_timestamp("EventTime")) LIMIT 10 OFFSET 1000;

‎hyper-parquet/results/c6a.4xlarge.json‎

Lines changed: 44 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"system": "Salesforce Hyper (Parquet)",
3-
"date": "2024-01-17",
3+
"date": "2025-03-01",
44
"machine": "c6a.4xlarge, 500gb gp2",
55
"cluster_size": 1,
66

@@ -9,48 +9,48 @@
99
"load_time": 0,
1010
"data_size": 14737666736,
1111
"result": [
12-
[0.7219097170000168,0.027619491999757884,0.027920489000280213],
13-
[0.737111797999205,0.048634149999998044,0.047378079999361944],
14-
[0.9245716779996656,0.17131707599946822,0.16804028499973356],
15-
[1.6030505829994581,0.12902164600018295,0.12860479700066207],
16-
[2.0411895189999996,0.7501233949997186,0.7229265850000957],
17-
[2.214053431999673,0.42726217600011296,0.41227727600016806],
18-
[0.8370674489997327,0.10616081199987093,0.10510917600004177],
19-
[0.7842502269995748,0.05004716100029327,0.04745348800042848],
20-
[2.378508595999847,0.9582314529998257,0.9595492039998135],
21-
[2.98738310599947,1.1077083719992515,1.103264682000372],
22-
[1.7991438899998684,0.13047813800039876,0.130802247000247],
23-
[1.8097300629997335,0.1563592239999707,0.15264345299965498],
24-
[2.2247359569992113,0.4469202050004242,0.4600646389999383],
25-
[4.190823522999381,1.335163678999379,1.3213064569999915],
26-
[2.284668049999709,0.47372808799991617,0.4722525119996135],
27-
[2.0657329010000467,0.7685637869999482,0.7594056219995764],
28-
[3.8420034400005534,1.1448920030006775,1.1493486809995375],
29-
[3.6921915029997763,1.0196109890002845,1.0164555909996125],
30-
[6.230567397999948,2.579583086000639,2.5722333799994885],
31-
[1.3792135950006923,0.059864528999241884,0.06061175699960586],
32-
[10.693764493999879,1.7598096270003225,1.7332925750006325],
33-
[11.190589746999649,1.6949993620000896,1.7555093920000218],
34-
[14.150610290000259,1.773250088000168,1.7303812159998415],
35-
[27.00777113500044,1.834026033000555,1.8222103819998665],
36-
[3.860901016999378,0.2714062750001176,0.2638269600001877],
37-
[2.0005952850005997,0.18801778499982902,0.1782568770004218],
38-
[3.858351470999878,0.27515161500014074,0.27319643600003474],
39-
[10.848775522999858,3.0039038149998305,2.929092249999485],
40-
[9.987565907000317,4.583312093000131,4.5697411940000165],
41-
[5.256411803999981,4.49008911600049,4.534523643999819],
42-
[3.880485081000188,0.5382274960002178,0.5422821440006373],
43-
[7.52864157499971,0.810706104000019,0.7913294809995932],
44-
[7.951601234000009,4.221059950000381,4.216031142999782],
45-
[11.226983931000177,3.16029577900008,3.24800152699936],
46-
[11.213096481999855,3.213449804999982,3.243356124999991],
47-
[1.6867397890000575,0.7003687099995659,0.7037680719995478],
48-
[0.8482456660003663,0.019971596000686986,0.021713408999858075],
49-
[0.8555035409999618,0.019188197000403306,0.020169281000562478],
50-
[0.8434355159997722,0.016406038999775774,0.015724162999504188],
51-
[0.8927955239996663,0.0192574489992694,0.019196966999516007],
52-
[0.7656367609997687,0.018199842999820248,0.01630677600041963],
53-
[0.7545674700004383,0.017830903999310976,0.01580502499928116],
54-
[0.7797055289993295,0.02250340699993103,0.019546015999367228]
12+
[0.6985541139999896,0.027216920000000755,0.02855089299998781],
13+
[0.6940044039999975,0.04860826000003726,0.049151961999996274],
14+
[0.8971933960000342,0.17664967700000034,0.1702298420000261],
15+
[1.6075812670000005,0.12920780600006765,0.12882554700001947],
16+
[2.0165824840000823,0.724728339999956,0.742897871000082],
17+
[2.1868178959999796,0.42228560900002776,0.41470915599995806],
18+
[0.7550829959999419,0.1037783890000128,0.10456691800004592],
19+
[0.7008819010000025,0.05105827900001714,0.04610908899996957],
20+
[2.3398681300000135,0.9372953990000497,0.9594750869999871],
21+
[2.9709995289999824,1.1059870939999428,1.0968814620000558],
22+
[1.79800228900001,0.13538952600003995,0.12795187599999736],
23+
[1.804587988000094,0.15226294699994014,0.15126008899994758],
24+
[2.22624754200001,0.4370266180000044,0.44781985700001314],
25+
[4.178947770000036,1.3337199710000505,1.3380890769999496],
26+
[2.2655287640000097,0.49567368199996054,0.48954709800000273],
27+
[2.067545822999932,0.7794534509999949,0.7788017039999886],
28+
[3.8150172480000037,1.124171179999962,1.13999706900006],
29+
[3.7510353590000705,1.0424183430000085,1.0304538410000532],
30+
[6.23214877700002,2.625024096000061,2.6283640130000094],
31+
[1.3714859949999436,0.058160605000011856,0.05800351100003809],
32+
[10.62139656699992,1.6995983519999527,1.7134582910000518],
33+
[11.258012755000095,1.7273341959999016,1.7680606559999887],
34+
[14.178312829000106,1.711714819000008,1.720642923000014],
35+
[26.960378805000005,1.8026549179999165,1.796680316999982],
36+
[3.8539238529999693,0.2625974090000227,0.26623333700001695],
37+
[2.005475039999965,0.18303656499995213,0.1786459960000002],
38+
[3.8546983010000986,0.27258231700000124,0.27733553299992764],
39+
[10.876402912999993,3.0302669819999437,3.0771695339999496],
40+
[9.971713970999986,4.639474280000059,4.58498229099996],
41+
[5.143137952000075,4.482304221000163,4.589473950999945],
42+
[3.8281821759999275,0.5546492420000959,0.5367887190000147],
43+
[7.5201330689999395,0.8570178140000735,0.8579126929998893],
44+
[8.042442435000112,4.483592042000055,4.425752202000012],
45+
[11.182898210000076,3.1866615869998896,3.1669316750001144],
46+
[11.228322655999818,3.178803332000143,3.1693062730000747],
47+
[1.6784054899999319,0.7295663160000458,0.7088829439999245],
48+
[0.8270137200001955,0.13063479800007372,0.11806330200010962],
49+
[0.7335704600000099,0.0433581879999565,0.04540310099991984],
50+
[0.7780583579999529,0.03800044800004798,0.03654643000004398],
51+
[0.9612860100000944,0.1585096279998197,0.16010026000003563],
52+
[0.6888319909999154,0.0291583170001104,0.02186890799998764],
53+
[0.8061543610001536,0.02377491800007192,0.024389062999944144],
54+
[0.6877752000000328,0.04160301900014929,0.04005347999986952]
5555
]
5656
}

‎hyper-parquet/results/c6a.metal.json‎

Lines changed: 44 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"system": "Salesforce Hyper (Parquet)",
3-
"date": "2024-01-17",
3+
"date": "2025-03-01",
44
"machine": "c6a.metal, 500gb gp2",
55
"cluster_size": 1,
66

@@ -9,48 +9,48 @@
99
"load_time": 0,
1010
"data_size": 14737666736,
1111
"result": [
12-
[0.6889787889995205,0.013737944000240532,0.014937410000129603],
13-
[0.7034498949997214,0.021791494999888528,0.019727671999135055],
14-
[0.8999540049999268,0.04373378499985847,0.03797985199980758],
15-
[1.605964244000461,0.04207935000067664,0.0411699710002722],
16-
[1.7158774749996155,0.15103437800007669,0.15754237700002705],
17-
[2.063965831999667,0.1444641660000343,0.14903264399981708],
18-
[0.7156979090004825,0.02909788099987054,0.024683267000000342],
19-
[0.6884307640002589,0.02407470399975864,0.023058033000779687],
20-
[1.999640148999788,0.2288032549995478,0.2234107699996457],
21-
[2.5972356709999076,0.24009356500027934,0.23674606300028245],
22-
[1.7926683440000488,0.05801006799993047,0.05775474299935013],
23-
[1.7968784639997466,0.06412765800087072,0.06335002099967824],
24-
[2.062435737000669,0.14288391699938074,0.14384252800027753],
25-
[3.6724959570001374,0.47648941699935676,0.5449808370003666],
26-
[2.1148565089997646,0.1539659109994318,0.16226776799976506],
27-
[1.715407175999644,0.16263590400012617,0.1701330349997079],
28-
[3.4959975070005385,0.33218501300052594,0.3427060879994315],
29-
[3.433724799000629,0.2828800309998769,0.27985662700029934],
30-
[5.532748878999882,0.6284413600005792,0.6183132030000706],
31-
[1.3772007019997545,0.03982739700040838,0.04161049499998626],
32-
[10.625532039000063,0.44653291899976466,0.4204516350000631],
33-
[11.137092470000425,0.43344174199955887,0.39003461100037384],
34-
[14.184794642999805,0.37042895699960354,0.3811335930004134],
35-
[27.016760841000178,0.5693049620003876,0.6119619350001813],
36-
[3.8582435849994,0.100818383000842,0.10260137099976419],
37-
[2.010061078000035,0.06726427300054638,0.06484739200004697],
38-
[3.864138844000081,0.1064601620000758,0.09976519000065309],
39-
[10.773716523000076,0.6136059179998483,0.6020529329998681],
40-
[9.697728103000372,1.416506214999572,1.5978383809997467],
41-
[1.4001344609996522,0.6825155389997235,0.607482133999838],
42-
[3.673145996999665,0.16566586599947186,0.1702915440000652],
43-
[7.26117624500057,0.26332993500000157,0.2727121919997444],
44-
[6.178020389999801,0.8436634179997782,0.9017057499995644],
45-
[10.972262789000524,0.8583406440002364,0.84962970900051],
46-
[10.999648767000508,0.9086391500004538,0.9373917439997967],
47-
[1.3885262369994962,0.1385416859993711,0.13993194600061543],
48-
[0.7878563610001947,0.021241037000436336,0.02111566399980802],
49-
[0.7794140409996544,0.019473798999570135,0.020300008000049274],
50-
[0.7934780760006106,0.01980310599992663,0.017497969000032754],
51-
[0.8085245000002033,0.023551405999569397,0.021493882999493508],
52-
[0.7324483279999185,0.01910419199975877,0.018958818999635696],
53-
[0.6993875810003374,0.016606039000180317,0.019143223000355647],
54-
[0.7051875810002457,0.022419511999942188,0.018006359000537486]
12+
[0.6572793919999924,0.014372903000008819,0.014968368000012333],
13+
[0.6803374360000021,0.02253393299997697,0.01931356800000117],
14+
[0.8963037569999983,0.04882953300000281,0.04778341900001237],
15+
[1.6087412280000137,0.04378158700001222,0.04385687800001392],
16+
[1.8351422929999899,0.14815896800001838,0.13875419900000452],
17+
[2.113131562999996,0.13056353900000772,0.1304780969999797],
18+
[0.6894404210000005,0.029125475999990158,0.0375497310000128],
19+
[0.6901104050000129,0.020827533000016274,0.01940006000000949],
20+
[2.010278193000005,0.23537161700002684,0.22228438400000528],
21+
[2.5778186980000157,0.22890824600000315,0.23005542300001025],
22+
[1.7929500529999984,0.05721926699999358,0.05460573499999555],
23+
[1.8009442960000115,0.05729837299998053,0.05346564000001308],
24+
[2.0632373849999794,0.15033445999995365,0.1496056889999977],
25+
[3.6477737490000095,0.5058042279999881,0.49572334300000875],
26+
[2.115116657000044,0.14796877300000233,0.14973598000000266],
27+
[1.7294786990000262,0.1755568799999878,0.1697169550000126],
28+
[3.515351593000048,0.34414215599997533,0.3501185449999866],
29+
[3.4048882989999925,0.2281125239999824,0.2214414169999941],
30+
[5.482805406000011,0.5748405699999921,0.5526460329999736],
31+
[1.373404866000044,0.03916799199998877,0.035665398999981335],
32+
[10.66069594000004,0.4124743790000025,0.4171336989999759],
33+
[11.169437373999983,0.40411928200001057,0.3933232910000015],
34+
[14.159519891000002,0.358043825999971,0.34997903700002553],
35+
[27.014248142999975,0.617882437999981,0.5496720350000146],
36+
[3.8619844700000385,0.09870847199999844,0.10183087100000421],
37+
[2.006016676999991,0.06430414499999415,0.06280953599997474],
38+
[3.8656452859999604,0.11297104499999477,0.10386616400001003],
39+
[10.773004632000038,0.5955276839999897,0.6082511539999587],
40+
[9.766736315999992,1.465654298000004,1.5062321119999638],
41+
[1.3552058369999713,0.5877868059999969,0.5667613449999749],
42+
[3.6565027419999865,0.15670384600002762,0.17305791299997964],
43+
[7.278817306000008,0.28676614300002257,0.30634865799999034],
44+
[6.047333771000012,0.8262016209999956,0.8236209190000068],
45+
[10.97555794599998,0.8898764340000298,0.9394563040000321],
46+
[11.020369867999989,0.9087671909999813,0.8979746960000057],
47+
[1.3828137689999949,0.13577607400003444,0.13134211499999537],
48+
[0.884538707000047,0.13435938999998598,0.1290491740000448],
49+
[0.7520538169999895,0.05126737799997727,0.047700327000029574],
50+
[0.7670295390000206,0.04348828299998786,0.04306924499996967],
51+
[0.9724227699999801,0.15415145999998003,0.16159423900001002],
52+
[0.7285194280000269,0.029187917999990987,0.029708839000022635],
53+
[0.8232950490000235,0.023710457999982282,0.022837490999961574],
54+
[0.7612474169999928,0.04576240399995868,0.04463043200001948]
5555
]
5656
}

0 commit comments

Comments
 (0)