Skip to content

Commit 4d1c6cd

Browse files
Caetano Sauercaetanosauer
Caetano Sauer
authored and
caetanosauer
committed
Hyper: Fix EventTime handling in Parquet
The Parquet queries in Hyper were applying the `to_timestamp` function to convert an integer to a timestamp, but this was incomplete because that function takes the number of seconds since Unix epoch, and not the number of days as stored in the EventDate column. To fix that, we now use an explicit addition from the Unix epoch. This makes our results slightly slower but correct. Kudos to Mark Raasveldt for reporting the problem.
1 parent bfd5283 commit 4d1c6cd

File tree

3 files changed

+96
-96
lines changed

3 files changed

+96
-96
lines changed

‎hyper-parquet/queries.sql‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ SELECT SUM("AdvEngineID"), COUNT(*), AVG("ResolutionWidth") FROM hits;
44
SELECT AVG("UserID") FROM hits;
55
SELECT COUNT(DISTINCT "UserID") FROM hits;
66
SELECT COUNT(DISTINCT "SearchPhrase") FROM hits;
7-
SELECT to_timestamp(MIN("EventDate")), to_timestamp(MAX("EventDate")) FROM hits;
7+
SELECT (date '1970-01-01' + MIN("EventDate")), (date '1970-01-01' + MAX("EventDate")) FROM hits;
88
SELECT "AdvEngineID", COUNT(*) FROM hits WHERE "AdvEngineID" <> 0 GROUP BY "AdvEngineID" ORDER BY COUNT(*) DESC;
99
SELECT "RegionID", COUNT(DISTINCT "UserID") AS u FROM hits GROUP BY "RegionID" ORDER BY u DESC LIMIT 10;
1010
SELECT "RegionID", SUM("AdvEngineID"), COUNT(*) AS c, AVG("ResolutionWidth"), COUNT(DISTINCT "UserID") FROM hits GROUP BY "RegionID" ORDER BY c DESC LIMIT 10;
@@ -34,10 +34,10 @@ SELECT "WatchID", "ClientIP", COUNT(*) AS c, SUM("IsRefresh"), AVG("ResolutionWi
3434
SELECT cast("URL" as text), COUNT(*) AS c FROM hits GROUP BY cast("URL" as text) ORDER BY c DESC LIMIT 10;
3535
SELECT 1, cast("URL" as text), COUNT(*) AS c FROM hits GROUP BY 1, cast("URL" as text) ORDER BY c DESC LIMIT 10;
3636
SELECT "ClientIP", "ClientIP" - 1, "ClientIP" - 2, "ClientIP" - 3, COUNT(*) AS c FROM hits GROUP BY "ClientIP", "ClientIP" - 1, "ClientIP" - 2, "ClientIP" - 3 ORDER BY c DESC LIMIT 10;
37-
SELECT "URL", COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate") >= '2013-07-01' AND to_timestamp("EventDate") <= '2013-07-31' AND "DontCountHits" = 0 AND "IsRefresh" = 0 AND cast("URL" as text) <> '' GROUP BY "URL" ORDER BY "PageViews" DESC LIMIT 10;
38-
SELECT "Title", COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate") >= '2013-07-01' AND to_timestamp("EventDate") <= '2013-07-31' AND "DontCountHits" = 0 AND "IsRefresh" = 0 AND "Title" <> '' GROUP BY "Title" ORDER BY "PageViews" DESC LIMIT 10;
39-
SELECT "URL", COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate") >= '2013-07-01' AND to_timestamp("EventDate") <= '2013-07-31' AND "IsRefresh" = 0 AND "IsLink" <> 0 AND "IsDownload" = 0 GROUP BY "URL" ORDER BY "PageViews" DESC LIMIT 10 OFFSET 1000;
40-
SELECT "TraficSourceID", "SearchEngineID", "AdvEngineID", CASE WHEN ("SearchEngineID" = 0 AND "AdvEngineID" = 0) THEN "Referer" ELSE '' END AS Src, "URL" AS Dst, COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate") >= '2013-07-01' AND to_timestamp("EventDate") <= '2013-07-31' AND "IsRefresh" = 0 GROUP BY "TraficSourceID", "SearchEngineID", "AdvEngineID", Src, Dst ORDER BY "PageViews" DESC LIMIT 10 OFFSET 1000;
41-
SELECT "URLHash", "EventDate", COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate") >= '2013-07-01' AND to_timestamp("EventDate") <= '2013-07-31' AND "IsRefresh" = 0 AND "TraficSourceID" IN (-1, 6) AND "RefererHash" = 3594120000172545465 GROUP BY "URLHash", "EventDate" ORDER BY "PageViews" DESC LIMIT 10 OFFSET 100;
42-
SELECT "WindowClientWidth", "WindowClientHeight", COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate") >= '2013-07-01' AND to_timestamp("EventDate") <= '2013-07-31' AND "IsRefresh" = 0 AND "DontCountHits" = 0 AND "URLHash" = 2868770270353813622 GROUP BY "WindowClientWidth", "WindowClientHeight" ORDER BY "PageViews" DESC LIMIT 10 OFFSET 10000;
43-
SELECT DATE_TRUNC('minute', to_timestamp("EventTime")) AS M, COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND to_timestamp("EventDate") >= '2013-07-14' AND to_timestamp("EventDate") <= '2013-07-15' AND "IsRefresh" = 0 AND "DontCountHits" = 0 GROUP BY DATE_TRUNC('minute', to_timestamp("EventTime")) ORDER BY DATE_TRUNC('minute', to_timestamp("EventTime")) LIMIT 10 OFFSET 1000;
37+
SELECT "URL", COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND (date '1970-01-01' + "EventDate") >= '2013-07-01' AND (date '1970-01-01' + "EventDate") <= '2013-07-31' AND "DontCountHits" = 0 AND "IsRefresh" = 0 AND cast("URL" as text) <> '' GROUP BY "URL" ORDER BY "PageViews" DESC LIMIT 10;
38+
SELECT "Title", COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND (date '1970-01-01' + "EventDate") >= '2013-07-01' AND (date '1970-01-01' + "EventDate") <= '2013-07-31' AND "DontCountHits" = 0 AND "IsRefresh" = 0 AND "Title" <> '' GROUP BY "Title" ORDER BY "PageViews" DESC LIMIT 10;
39+
SELECT "URL", COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND (date '1970-01-01' + "EventDate") >= '2013-07-01' AND (date '1970-01-01' + "EventDate") <= '2013-07-31' AND "IsRefresh" = 0 AND "IsLink" <> 0 AND "IsDownload" = 0 GROUP BY "URL" ORDER BY "PageViews" DESC LIMIT 10 OFFSET 1000;
40+
SELECT "TraficSourceID", "SearchEngineID", "AdvEngineID", CASE WHEN ("SearchEngineID" = 0 AND "AdvEngineID" = 0) THEN "Referer" ELSE '' END AS Src, "URL" AS Dst, COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND (date '1970-01-01' + "EventDate") >= '2013-07-01' AND (date '1970-01-01' + "EventDate") <= '2013-07-31' AND "IsRefresh" = 0 GROUP BY "TraficSourceID", "SearchEngineID", "AdvEngineID", Src, Dst ORDER BY "PageViews" DESC LIMIT 10 OFFSET 1000;
41+
SELECT "URLHash", (date '1970-01-01' + "EventDate"), COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND (date '1970-01-01' + "EventDate") >= '2013-07-01' AND (date '1970-01-01' + "EventDate") <= '2013-07-31' AND "IsRefresh" = 0 AND "TraficSourceID" IN (-1, 6) AND "RefererHash" = 3594120000172545465 GROUP BY "URLHash", (date '1970-01-01' + "EventDate") ORDER BY "PageViews" DESC LIMIT 10 OFFSET 100;
42+
SELECT "WindowClientWidth", "WindowClientHeight", COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND (date '1970-01-01' + "EventDate") >= '2013-07-01' AND (date '1970-01-01' + "EventDate") <= '2013-07-31' AND "IsRefresh" = 0 AND "DontCountHits" = 0 AND "URLHash" = 2868770270353813622 GROUP BY "WindowClientWidth", "WindowClientHeight" ORDER BY "PageViews" DESC LIMIT 10 OFFSET 10000;
43+
SELECT DATE_TRUNC('minute', to_timestamp("EventTime")) AS M, COUNT(*) AS "PageViews" FROM hits WHERE "CounterID" = 62 AND (date '1970-01-01' + "EventDate") >= '2013-07-14' AND (date '1970-01-01' + "EventDate") <= '2013-07-15' AND "IsRefresh" = 0 AND "DontCountHits" = 0 GROUP BY DATE_TRUNC('minute', to_timestamp("EventTime")) ORDER BY DATE_TRUNC('minute', to_timestamp("EventTime")) LIMIT 10 OFFSET 1000;

‎hyper-parquet/results/c6a.4xlarge.json‎

Lines changed: 44 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"system": "Salesforce Hyper (Parquet)",
3-
"date": "2024-01-17",
3+
"date": "2025-03-01",
44
"machine": "c6a.4xlarge, 500gb gp2",
55
"cluster_size": 1,
66

@@ -9,48 +9,48 @@
99
"load_time": 0,
1010
"data_size": 14737666736,
1111
"result": [
12-
[0.7219097170000168,0.027619491999757884,0.027920489000280213],
13-
[0.737111797999205,0.048634149999998044,0.047378079999361944],
14-
[0.9245716779996656,0.17131707599946822,0.16804028499973356],
15-
[1.6030505829994581,0.12902164600018295,0.12860479700066207],
16-
[2.0411895189999996,0.7501233949997186,0.7229265850000957],
17-
[2.214053431999673,0.42726217600011296,0.41227727600016806],
18-
[0.8370674489997327,0.10616081199987093,0.10510917600004177],
19-
[0.7842502269995748,0.05004716100029327,0.04745348800042848],
20-
[2.378508595999847,0.9582314529998257,0.9595492039998135],
21-
[2.98738310599947,1.1077083719992515,1.103264682000372],
22-
[1.7991438899998684,0.13047813800039876,0.130802247000247],
23-
[1.8097300629997335,0.1563592239999707,0.15264345299965498],
24-
[2.2247359569992113,0.4469202050004242,0.4600646389999383],
25-
[4.190823522999381,1.335163678999379,1.3213064569999915],
26-
[2.284668049999709,0.47372808799991617,0.4722525119996135],
27-
[2.0657329010000467,0.7685637869999482,0.7594056219995764],
28-
[3.8420034400005534,1.1448920030006775,1.1493486809995375],
29-
[3.6921915029997763,1.0196109890002845,1.0164555909996125],
30-
[6.230567397999948,2.579583086000639,2.5722333799994885],
31-
[1.3792135950006923,0.059864528999241884,0.06061175699960586],
32-
[10.693764493999879,1.7598096270003225,1.7332925750006325],
33-
[11.190589746999649,1.6949993620000896,1.7555093920000218],
34-
[14.150610290000259,1.773250088000168,1.7303812159998415],
35-
[27.00777113500044,1.834026033000555,1.8222103819998665],
36-
[3.860901016999378,0.2714062750001176,0.2638269600001877],
37-
[2.0005952850005997,0.18801778499982902,0.1782568770004218],
38-
[3.858351470999878,0.27515161500014074,0.27319643600003474],
39-
[10.848775522999858,3.0039038149998305,2.929092249999485],
40-
[9.987565907000317,4.583312093000131,4.5697411940000165],
41-
[5.256411803999981,4.49008911600049,4.534523643999819],
42-
[3.880485081000188,0.5382274960002178,0.5422821440006373],
43-
[7.52864157499971,0.810706104000019,0.7913294809995932],
44-
[7.951601234000009,4.221059950000381,4.216031142999782],
45-
[11.226983931000177,3.16029577900008,3.24800152699936],
46-
[11.213096481999855,3.213449804999982,3.243356124999991],
47-
[1.6867397890000575,0.7003687099995659,0.7037680719995478],
48-
[0.8482456660003663,0.019971596000686986,0.021713408999858075],
49-
[0.8555035409999618,0.019188197000403306,0.020169281000562478],
50-
[0.8434355159997722,0.016406038999775774,0.015724162999504188],
51-
[0.8927955239996663,0.0192574489992694,0.019196966999516007],
52-
[0.7656367609997687,0.018199842999820248,0.01630677600041963],
53-
[0.7545674700004383,0.017830903999310976,0.01580502499928116],
54-
[0.7797055289993295,0.02250340699993103,0.019546015999367228]
12+
[0.6985541139999896,0.027216920000000755,0.02855089299998781],
13+
[0.6940044039999975,0.04860826000003726,0.049151961999996274],
14+
[0.8971933960000342,0.17664967700000034,0.1702298420000261],
15+
[1.6075812670000005,0.12920780600006765,0.12882554700001947],
16+
[2.0165824840000823,0.724728339999956,0.742897871000082],
17+
[2.1868178959999796,0.42228560900002776,0.41470915599995806],
18+
[0.7550829959999419,0.1037783890000128,0.10456691800004592],
19+
[0.7008819010000025,0.05105827900001714,0.04610908899996957],
20+
[2.3398681300000135,0.9372953990000497,0.9594750869999871],
21+
[2.9709995289999824,1.1059870939999428,1.0968814620000558],
22+
[1.79800228900001,0.13538952600003995,0.12795187599999736],
23+
[1.804587988000094,0.15226294699994014,0.15126008899994758],
24+
[2.22624754200001,0.4370266180000044,0.44781985700001314],
25+
[4.178947770000036,1.3337199710000505,1.3380890769999496],
26+
[2.2655287640000097,0.49567368199996054,0.48954709800000273],
27+
[2.067545822999932,0.7794534509999949,0.7788017039999886],
28+
[3.8150172480000037,1.124171179999962,1.13999706900006],
29+
[3.7510353590000705,1.0424183430000085,1.0304538410000532],
30+
[6.23214877700002,2.625024096000061,2.6283640130000094],
31+
[1.3714859949999436,0.058160605000011856,0.05800351100003809],
32+
[10.62139656699992,1.6995983519999527,1.7134582910000518],
33+
[11.258012755000095,1.7273341959999016,1.7680606559999887],
34+
[14.178312829000106,1.711714819000008,1.720642923000014],
35+
[26.960378805000005,1.8026549179999165,1.796680316999982],
36+
[3.8539238529999693,0.2625974090000227,0.26623333700001695],
37+
[2.005475039999965,0.18303656499995213,0.1786459960000002],
38+
[3.8546983010000986,0.27258231700000124,0.27733553299992764],
39+
[10.876402912999993,3.0302669819999437,3.0771695339999496],
40+
[9.971713970999986,4.639474280000059,4.58498229099996],
41+
[5.143137952000075,4.482304221000163,4.589473950999945],
42+
[3.8281821759999275,0.5546492420000959,0.5367887190000147],
43+
[7.5201330689999395,0.8570178140000735,0.8579126929998893],
44+
[8.042442435000112,4.483592042000055,4.425752202000012],
45+
[11.182898210000076,3.1866615869998896,3.1669316750001144],
46+
[11.228322655999818,3.178803332000143,3.1693062730000747],
47+
[1.6784054899999319,0.7295663160000458,0.7088829439999245],
48+
[0.8270137200001955,0.13063479800007372,0.11806330200010962],
49+
[0.7335704600000099,0.0433581879999565,0.04540310099991984],
50+
[0.7780583579999529,0.03800044800004798,0.03654643000004398],
51+
[0.9612860100000944,0.1585096279998197,0.16010026000003563],
52+
[0.6888319909999154,0.0291583170001104,0.02186890799998764],
53+
[0.8061543610001536,0.02377491800007192,0.024389062999944144],
54+
[0.6877752000000328,0.04160301900014929,0.04005347999986952]
5555
]
5656
}

‎hyper-parquet/results/c6a.metal.json‎

Lines changed: 44 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"system": "Salesforce Hyper (Parquet)",
3-
"date": "2024-01-17",
3+
"date": "2025-03-01",
44
"machine": "c6a.metal, 500gb gp2",
55
"cluster_size": 1,
66

@@ -9,48 +9,48 @@
99
"load_time": 0,
1010
"data_size": 14737666736,
1111
"result": [
12-
[0.6889787889995205,0.013737944000240532,0.014937410000129603],
13-
[0.7034498949997214,0.021791494999888528,0.019727671999135055],
14-
[0.8999540049999268,0.04373378499985847,0.03797985199980758],
15-
[1.605964244000461,0.04207935000067664,0.0411699710002722],
16-
[1.7158774749996155,0.15103437800007669,0.15754237700002705],
17-
[2.063965831999667,0.1444641660000343,0.14903264399981708],
18-
[0.7156979090004825,0.02909788099987054,0.024683267000000342],
19-
[0.6884307640002589,0.02407470399975864,0.023058033000779687],
20-
[1.999640148999788,0.2288032549995478,0.2234107699996457],
21-
[2.5972356709999076,0.24009356500027934,0.23674606300028245],
22-
[1.7926683440000488,0.05801006799993047,0.05775474299935013],
23-
[1.7968784639997466,0.06412765800087072,0.06335002099967824],
24-
[2.062435737000669,0.14288391699938074,0.14384252800027753],
25-
[3.6724959570001374,0.47648941699935676,0.5449808370003666],
26-
[2.1148565089997646,0.1539659109994318,0.16226776799976506],
27-
[1.715407175999644,0.16263590400012617,0.1701330349997079],
28-
[3.4959975070005385,0.33218501300052594,0.3427060879994315],
29-
[3.433724799000629,0.2828800309998769,0.27985662700029934],
30-
[5.532748878999882,0.6284413600005792,0.6183132030000706],
31-
[1.3772007019997545,0.03982739700040838,0.04161049499998626],
32-
[10.625532039000063,0.44653291899976466,0.4204516350000631],
33-
[11.137092470000425,0.43344174199955887,0.39003461100037384],
34-
[14.184794642999805,0.37042895699960354,0.3811335930004134],
35-
[27.016760841000178,0.5693049620003876,0.6119619350001813],
36-
[3.8582435849994,0.100818383000842,0.10260137099976419],
37-
[2.010061078000035,0.06726427300054638,0.06484739200004697],
38-
[3.864138844000081,0.1064601620000758,0.09976519000065309],
39-
[10.773716523000076,0.6136059179998483,0.6020529329998681],
40-
[9.697728103000372,1.416506214999572,1.5978383809997467],
41-
[1.4001344609996522,0.6825155389997235,0.607482133999838],
42-
[3.673145996999665,0.16566586599947186,0.1702915440000652],
43-
[7.26117624500057,0.26332993500000157,0.2727121919997444],
44-
[6.178020389999801,0.8436634179997782,0.9017057499995644],
45-
[10.972262789000524,0.8583406440002364,0.84962970900051],
46-
[10.999648767000508,0.9086391500004538,0.9373917439997967],
47-
[1.3885262369994962,0.1385416859993711,0.13993194600061543],
48-
[0.7878563610001947,0.021241037000436336,0.02111566399980802],
49-
[0.7794140409996544,0.019473798999570135,0.020300008000049274],
50-
[0.7934780760006106,0.01980310599992663,0.017497969000032754],
51-
[0.8085245000002033,0.023551405999569397,0.021493882999493508],
52-
[0.7324483279999185,0.01910419199975877,0.018958818999635696],
53-
[0.6993875810003374,0.016606039000180317,0.019143223000355647],
54-
[0.7051875810002457,0.022419511999942188,0.018006359000537486]
12+
[0.6572793919999924,0.014372903000008819,0.014968368000012333],
13+
[0.6803374360000021,0.02253393299997697,0.01931356800000117],
14+
[0.8963037569999983,0.04882953300000281,0.04778341900001237],
15+
[1.6087412280000137,0.04378158700001222,0.04385687800001392],
16+
[1.8351422929999899,0.14815896800001838,0.13875419900000452],
17+
[2.113131562999996,0.13056353900000772,0.1304780969999797],
18+
[0.6894404210000005,0.029125475999990158,0.0375497310000128],
19+
[0.6901104050000129,0.020827533000016274,0.01940006000000949],
20+
[2.010278193000005,0.23537161700002684,0.22228438400000528],
21+
[2.5778186980000157,0.22890824600000315,0.23005542300001025],
22+
[1.7929500529999984,0.05721926699999358,0.05460573499999555],
23+
[1.8009442960000115,0.05729837299998053,0.05346564000001308],
24+
[2.0632373849999794,0.15033445999995365,0.1496056889999977],
25+
[3.6477737490000095,0.5058042279999881,0.49572334300000875],
26+
[2.115116657000044,0.14796877300000233,0.14973598000000266],
27+
[1.7294786990000262,0.1755568799999878,0.1697169550000126],
28+
[3.515351593000048,0.34414215599997533,0.3501185449999866],
29+
[3.4048882989999925,0.2281125239999824,0.2214414169999941],
30+
[5.482805406000011,0.5748405699999921,0.5526460329999736],
31+
[1.373404866000044,0.03916799199998877,0.035665398999981335],
32+
[10.66069594000004,0.4124743790000025,0.4171336989999759],
33+
[11.169437373999983,0.40411928200001057,0.3933232910000015],
34+
[14.159519891000002,0.358043825999971,0.34997903700002553],
35+
[27.014248142999975,0.617882437999981,0.5496720350000146],
36+
[3.8619844700000385,0.09870847199999844,0.10183087100000421],
37+
[2.006016676999991,0.06430414499999415,0.06280953599997474],
38+
[3.8656452859999604,0.11297104499999477,0.10386616400001003],
39+
[10.773004632000038,0.5955276839999897,0.6082511539999587],
40+
[9.766736315999992,1.465654298000004,1.5062321119999638],
41+
[1.3552058369999713,0.5877868059999969,0.5667613449999749],
42+
[3.6565027419999865,0.15670384600002762,0.17305791299997964],
43+
[7.278817306000008,0.28676614300002257,0.30634865799999034],
44+
[6.047333771000012,0.8262016209999956,0.8236209190000068],
45+
[10.97555794599998,0.8898764340000298,0.9394563040000321],
46+
[11.020369867999989,0.9087671909999813,0.8979746960000057],
47+
[1.3828137689999949,0.13577607400003444,0.13134211499999537],
48+
[0.884538707000047,0.13435938999998598,0.1290491740000448],
49+
[0.7520538169999895,0.05126737799997727,0.047700327000029574],
50+
[0.7670295390000206,0.04348828299998786,0.04306924499996967],
51+
[0.9724227699999801,0.15415145999998003,0.16159423900001002],
52+
[0.7285194280000269,0.029187917999990987,0.029708839000022635],
53+
[0.8232950490000235,0.023710457999982282,0.022837490999961574],
54+
[0.7612474169999928,0.04576240399995868,0.04463043200001948]
5555
]
5656
}

0 commit comments

Comments
 (0)