diff --git a/enginetest/join_planning_tests.go b/enginetest/join_planning_tests.go index 753bbec61b..3fec5bc238 100644 --- a/enginetest/join_planning_tests.go +++ b/enginetest/join_planning_tests.go @@ -260,7 +260,7 @@ var JoinPlanningTests = []joinPlanScript{ { // When primary table is much larger, doing many lookups is expensive: prefer merge q: "select /*+ JOIN_ORDER(rs, xy) */ * from rs join xy on x = r order by 1,3", - types: []plan.JoinType{plan.JoinTypeLookup}, + types: []plan.JoinType{plan.JoinTypeMerge}, exp: []sql.Row{{0, 0, 0, 8}, {2, 3, 2, 1}, {3, 0, 3, 7}, {4, 8, 4, 0}, {5, 4, 5, 4}}, }, { @@ -411,7 +411,7 @@ var JoinPlanningTests = []joinPlanScript{ }, { q: "select * from xy where y+1 not in (select u from uv);", - types: []plan.JoinType{plan.JoinTypeLeftOuterHashExcludeNulls}, + types: []plan.JoinType{plan.JoinTypeLeftOuterLookup}, exp: []sql.Row{{3, 3}}, }, { @@ -822,7 +822,7 @@ where u in (select * from rec);`, tests: []JoinPlanTest{ { q: "select * from xy where x in (select u from uv join ab on u = a and a = 2) order by 1;", - types: []plan.JoinType{plan.JoinTypeInner, plan.JoinTypeLookup}, + types: []plan.JoinType{plan.JoinTypeLookup, plan.JoinTypeLookup}, exp: []sql.Row{{2, 1}}, }, { diff --git a/enginetest/queries/query_plans.go b/enginetest/queries/query_plans.go index 4999da870d..9a9fa580e9 100644 --- a/enginetest/queries/query_plans.go +++ b/enginetest/queries/query_plans.go @@ -1227,75 +1227,86 @@ WHERE " └─ GroupBy\n" + " ├─ select: COUNTDISTINCT([stock1.s_i_id])\n" + " ├─ group: \n" + - " └─ HashJoin\n" + + " └─ LookupJoin\n" + " ├─ Eq\n" + - " │ ├─ stock1.s_i_id:4!null\n" + - " │ └─ order_line1.ol_i_id:3\n" + - " ├─ IndexedTableAccess(order_line1)\n" + - " │ ├─ index: [order_line1.ol_w_id,order_line1.ol_d_id,order_line1.ol_o_id,order_line1.ol_number]\n" + - " │ ├─ static: [{[5, 5], [2, 2], [2981, 3001), [NULL, ∞)}]\n" + - " │ ├─ colSet: (1-10)\n" + - " │ ├─ tableId: 1\n" + - " │ └─ Table\n" + - " │ ├─ name: order_line1\n" + - " │ └─ columns: [ol_o_id ol_d_id ol_w_id ol_i_id]\n" + - " └─ HashLookup\n" + - " ├─ left-key: TUPLE(order_line1.ol_i_id:3)\n" + - " ├─ right-key: TUPLE(stock1.s_i_id:0!null)\n" + - " └─ Filter\n" + - " ├─ LessThan\n" + - " │ ├─ stock1.s_quantity:2\n" + - " │ └─ 15 (smallint)\n" + - " └─ IndexedTableAccess(stock1)\n" + - " ├─ index: [stock1.s_w_id,stock1.s_i_id]\n" + - " ├─ static: [{[5, 5], [NULL, ∞)}]\n" + - " ├─ colSet: (11-27)\n" + - " ├─ tableId: 2\n" + - " └─ Table\n" + - " ├─ name: stock1\n" + - " └─ columns: [s_i_id s_w_id s_quantity]\n" + + " │ ├─ stock1.s_i_id:0!null\n" + + " │ └─ order_line1.ol_i_id:6\n" + + " ├─ Filter\n" + + " │ ├─ LessThan\n" + + " │ │ ├─ stock1.s_quantity:2\n" + + " │ │ └─ 15 (smallint)\n" + + " │ └─ IndexedTableAccess(stock1)\n" + + " │ ├─ index: [stock1.s_w_id,stock1.s_i_id]\n" + + " │ ├─ static: [{[5, 5], [NULL, ∞)}]\n" + + " │ ├─ colSet: (11-27)\n" + + " │ ├─ tableId: 2\n" + + " │ └─ Table\n" + + " │ ├─ name: stock1\n" + + " │ └─ columns: [s_i_id s_w_id s_quantity]\n" + + " └─ Filter\n" + + " ├─ AND\n" + + " │ ├─ AND\n" + + " │ │ ├─ AND\n" + + " │ │ │ ├─ Eq\n" + + " │ │ │ │ ├─ order_line1.ol_w_id:2!null\n" + + " │ │ │ │ └─ 5 (smallint)\n" + + " │ │ │ └─ Eq\n" + + " │ │ │ ├─ order_line1.ol_d_id:1!null\n" + + " │ │ │ └─ 2 (tinyint)\n" + + " │ │ └─ LessThan\n" + + " │ │ ├─ order_line1.ol_o_id:0!null\n" + + " │ │ └─ 3001 (int)\n" + + " │ └─ GreaterThanOrEqual\n" + + " │ ├─ order_line1.ol_o_id:0!null\n" + + " │ └─ 2981 (int)\n" + + " └─ IndexedTableAccess(order_line1)\n" + + " ├─ index: [order_line1.ol_w_id,order_line1.ol_d_id,order_line1.ol_o_id,order_line1.ol_number]\n" + + " ├─ keys: [5 (smallint) 2 (tinyint)]\n" + + " ├─ colSet: (1-10)\n" + + " ├─ tableId: 1\n" + + " └─ Table\n" + + " ├─ name: order_line1\n" + + " └─ columns: [ol_o_id ol_d_id ol_w_id ol_i_id]\n" + "", ExpectedEstimates: "Project\n" + " ├─ columns: [countdistinct([stock1.s_i_id]) as COUNT(DISTINCT (s_i_id))]\n" + " └─ GroupBy\n" + " ├─ SelectedExprs(COUNTDISTINCT([stock1.s_i_id]))\n" + " ├─ Grouping()\n" + - " └─ HashJoin\n" + + " └─ LookupJoin\n" + " ├─ (stock1.s_i_id = order_line1.ol_i_id)\n" + - " ├─ IndexedTableAccess(order_line1)\n" + - " │ ├─ index: [order_line1.ol_w_id,order_line1.ol_d_id,order_line1.ol_o_id,order_line1.ol_number]\n" + - " │ ├─ filters: [{[5, 5], [2, 2], [2981, 3001), [NULL, ∞)}]\n" + - " │ └─ columns: [ol_o_id ol_d_id ol_w_id ol_i_id]\n" + - " └─ HashLookup\n" + - " ├─ left-key: (order_line1.ol_i_id)\n" + - " ├─ right-key: (stock1.s_i_id)\n" + - " └─ Filter\n" + - " ├─ (stock1.s_quantity < 15)\n" + - " └─ IndexedTableAccess(stock1)\n" + - " ├─ index: [stock1.s_w_id,stock1.s_i_id]\n" + - " ├─ filters: [{[5, 5], [NULL, ∞)}]\n" + - " └─ columns: [s_i_id s_w_id s_quantity]\n" + + " ├─ Filter\n" + + " │ ├─ (stock1.s_quantity < 15)\n" + + " │ └─ IndexedTableAccess(stock1)\n" + + " │ ├─ index: [stock1.s_w_id,stock1.s_i_id]\n" + + " │ ├─ filters: [{[5, 5], [NULL, ∞)}]\n" + + " │ └─ columns: [s_i_id s_w_id s_quantity]\n" + + " └─ Filter\n" + + " ├─ ((((order_line1.ol_w_id = 5) AND (order_line1.ol_d_id = 2)) AND (order_line1.ol_o_id < 3001)) AND (order_line1.ol_o_id >= 2981))\n" + + " └─ IndexedTableAccess(order_line1)\n" + + " ├─ index: [order_line1.ol_w_id,order_line1.ol_d_id,order_line1.ol_o_id,order_line1.ol_number]\n" + + " ├─ columns: [ol_o_id ol_d_id ol_w_id ol_i_id]\n" + + " └─ keys: 5, 2\n" + "", ExpectedAnalysis: "Project\n" + " ├─ columns: [countdistinct([stock1.s_i_id]) as COUNT(DISTINCT (s_i_id))]\n" + " └─ GroupBy\n" + " ├─ SelectedExprs(COUNTDISTINCT([stock1.s_i_id]))\n" + " ├─ Grouping()\n" + - " └─ HashJoin\n" + + " └─ LookupJoin\n" + " ├─ (stock1.s_i_id = order_line1.ol_i_id)\n" + - " ├─ IndexedTableAccess(order_line1)\n" + - " │ ├─ index: [order_line1.ol_w_id,order_line1.ol_d_id,order_line1.ol_o_id,order_line1.ol_number]\n" + - " │ ├─ filters: [{[5, 5], [2, 2], [2981, 3001), [NULL, ∞)}]\n" + - " │ └─ columns: [ol_o_id ol_d_id ol_w_id ol_i_id]\n" + - " └─ HashLookup\n" + - " ├─ left-key: (order_line1.ol_i_id)\n" + - " ├─ right-key: (stock1.s_i_id)\n" + - " └─ Filter\n" + - " ├─ (stock1.s_quantity < 15)\n" + - " └─ IndexedTableAccess(stock1)\n" + - " ├─ index: [stock1.s_w_id,stock1.s_i_id]\n" + - " ├─ filters: [{[5, 5], [NULL, ∞)}]\n" + - " └─ columns: [s_i_id s_w_id s_quantity]\n" + + " ├─ Filter\n" + + " │ ├─ (stock1.s_quantity < 15)\n" + + " │ └─ IndexedTableAccess(stock1)\n" + + " │ ├─ index: [stock1.s_w_id,stock1.s_i_id]\n" + + " │ ├─ filters: [{[5, 5], [NULL, ∞)}]\n" + + " │ └─ columns: [s_i_id s_w_id s_quantity]\n" + + " └─ Filter\n" + + " ├─ ((((order_line1.ol_w_id = 5) AND (order_line1.ol_d_id = 2)) AND (order_line1.ol_o_id < 3001)) AND (order_line1.ol_o_id >= 2981))\n" + + " └─ IndexedTableAccess(order_line1)\n" + + " ├─ index: [order_line1.ol_w_id,order_line1.ol_d_id,order_line1.ol_o_id,order_line1.ol_number]\n" + + " ├─ columns: [ol_o_id ol_d_id ol_w_id ol_i_id]\n" + + " └─ keys: 5, 2\n" + "", }, { @@ -1439,7 +1450,13 @@ where ExpectedPlan: "Project\n" + " ├─ columns: [style.assetId:1]\n" + " └─ LookupJoin\n" + + " ├─ Eq\n" + + " │ ├─ style.assetId:1\n" + + " │ └─ color.assetId:9\n" + " ├─ LookupJoin\n" + + " │ ├─ Eq\n" + + " │ │ ├─ style.assetId:1\n" + + " │ │ └─ dimension.assetId:5\n" + " │ ├─ TableAlias(style)\n" + " │ │ └─ IndexedTableAccess(asset)\n" + " │ │ ├─ index: [asset.orgId,asset.name,asset.val]\n" + @@ -1463,8 +1480,8 @@ where " │ │ └─ org1 (longtext)\n" + " │ └─ TableAlias(dimension)\n" + " │ └─ IndexedTableAccess(asset)\n" + - " │ ├─ index: [asset.orgId,asset.name,asset.assetId]\n" + - " │ ├─ keys: [org1 (longtext) dimension (longtext) style.assetId:1]\n" + + " │ ├─ index: [asset.orgId,asset.name,asset.val]\n" + + " │ ├─ keys: [org1 (longtext) dimension (longtext) wide (longtext)]\n" + " │ ├─ colSet: (6-10)\n" + " │ ├─ tableId: 2\n" + " │ └─ Table\n" + @@ -1484,8 +1501,8 @@ where " │ └─ org1 (longtext)\n" + " └─ TableAlias(color)\n" + " └─ IndexedTableAccess(asset)\n" + - " ├─ index: [asset.orgId,asset.name,asset.assetId]\n" + - " ├─ keys: [org1 (longtext) color (longtext) style.assetId:1]\n" + + " ├─ index: [asset.orgId,asset.name,asset.val]\n" + + " ├─ keys: [org1 (longtext) color (longtext) blue (longtext)]\n" + " ├─ colSet: (11-15)\n" + " ├─ tableId: 3\n" + " └─ Table\n" + @@ -1494,8 +1511,10 @@ where "", ExpectedEstimates: "Project\n" + " ├─ columns: [style.assetId]\n" + - " └─ LookupJoin (estimated cost=19.800 rows=6)\n" + - " ├─ LookupJoin (estimated cost=19.800 rows=6)\n" + + " └─ LookupJoin (estimated cost=18.900 rows=6)\n" + + " ├─ (style.assetId = color.assetId)\n" + + " ├─ LookupJoin (estimated cost=18.900 rows=6)\n" + + " │ ├─ (style.assetId = dimension.assetId)\n" + " │ ├─ TableAlias(style)\n" + " │ │ └─ IndexedTableAccess(asset)\n" + " │ │ ├─ index: [asset.orgId,asset.name,asset.val]\n" + @@ -1505,21 +1524,23 @@ where " │ ├─ (((dimension.val = 'wide') AND (dimension.name = 'dimension')) AND (dimension.orgId = 'org1'))\n" + " │ └─ TableAlias(dimension)\n" + " │ └─ IndexedTableAccess(asset)\n" + - " │ ├─ index: [asset.orgId,asset.name,asset.assetId]\n" + + " │ ├─ index: [asset.orgId,asset.name,asset.val]\n" + " │ ├─ columns: [orgid assetid name val]\n" + - " │ └─ keys: 'org1', 'dimension', style.assetId\n" + + " │ └─ keys: 'org1', 'dimension', 'wide'\n" + " └─ Filter\n" + " ├─ (((color.val = 'blue') AND (color.name = 'color')) AND (color.orgId = 'org1'))\n" + " └─ TableAlias(color)\n" + " └─ IndexedTableAccess(asset)\n" + - " ├─ index: [asset.orgId,asset.name,asset.assetId]\n" + + " ├─ index: [asset.orgId,asset.name,asset.val]\n" + " ├─ columns: [orgid assetid name val]\n" + - " └─ keys: 'org1', 'color', style.assetId\n" + + " └─ keys: 'org1', 'color', 'blue'\n" + "", ExpectedAnalysis: "Project\n" + " ├─ columns: [style.assetId]\n" + - " └─ LookupJoin (estimated cost=19.800 rows=6) (actual rows=1 loops=1)\n" + - " ├─ LookupJoin (estimated cost=19.800 rows=6) (actual rows=1 loops=1)\n" + + " └─ LookupJoin (estimated cost=18.900 rows=6) (actual rows=1 loops=1)\n" + + " ├─ (style.assetId = color.assetId)\n" + + " ├─ LookupJoin (estimated cost=18.900 rows=6) (actual rows=1 loops=1)\n" + + " │ ├─ (style.assetId = dimension.assetId)\n" + " │ ├─ TableAlias(style)\n" + " │ │ └─ IndexedTableAccess(asset)\n" + " │ │ ├─ index: [asset.orgId,asset.name,asset.val]\n" + @@ -1529,16 +1550,16 @@ where " │ ├─ (((dimension.val = 'wide') AND (dimension.name = 'dimension')) AND (dimension.orgId = 'org1'))\n" + " │ └─ TableAlias(dimension)\n" + " │ └─ IndexedTableAccess(asset)\n" + - " │ ├─ index: [asset.orgId,asset.name,asset.assetId]\n" + + " │ ├─ index: [asset.orgId,asset.name,asset.val]\n" + " │ ├─ columns: [orgid assetid name val]\n" + - " │ └─ keys: 'org1', 'dimension', style.assetId\n" + + " │ └─ keys: 'org1', 'dimension', 'wide'\n" + " └─ Filter\n" + " ├─ (((color.val = 'blue') AND (color.name = 'color')) AND (color.orgId = 'org1'))\n" + " └─ TableAlias(color)\n" + " └─ IndexedTableAccess(asset)\n" + - " ├─ index: [asset.orgId,asset.name,asset.assetId]\n" + + " ├─ index: [asset.orgId,asset.name,asset.val]\n" + " ├─ columns: [orgid assetid name val]\n" + - " └─ keys: 'org1', 'color', style.assetId\n" + + " └─ keys: 'org1', 'color', 'blue'\n" + "", }, { @@ -2721,21 +2742,14 @@ Select * from ( ExpectedPlan: "Project\n" + " ├─ columns: [rs.r:0!null, rs.s:1, xy.x:2!null, xy.y:3]\n" + " └─ Sort(rs.r:0!null ASC nullsFirst, xy.x:2!null ASC nullsFirst)\n" + - " └─ LeftOuterMergeJoin\n" + - " ├─ cmp: Eq\n" + - " │ ├─ rs.s:1\n" + - " │ └─ xy.y:3\n" + - " ├─ IndexedTableAccess(rs)\n" + - " │ ├─ index: [rs.s]\n" + - " │ ├─ static: [{[NULL, ∞)}]\n" + - " │ ├─ colSet: (1,2)\n" + - " │ ├─ tableId: 1\n" + + " └─ LeftOuterLookupJoin\n" + + " ├─ ProcessTable\n" + " │ └─ Table\n" + " │ ├─ name: rs\n" + " │ └─ columns: [r s]\n" + " └─ IndexedTableAccess(xy)\n" + " ├─ index: [xy.y]\n" + - " ├─ static: [{[NULL, ∞)}]\n" + + " ├─ keys: [rs.s:1]\n" + " ├─ colSet: (3,4)\n" + " ├─ tableId: 2\n" + " └─ Table\n" + @@ -2745,30 +2759,26 @@ Select * from ( ExpectedEstimates: "Project\n" + " ├─ columns: [rs.r, rs.s, xy.x, xy.y]\n" + " └─ Sort(rs.r ASC, xy.x ASC)\n" + - " └─ LeftOuterMergeJoin\n" + - " ├─ cmp: (rs.s = xy.y)\n" + - " ├─ IndexedTableAccess(rs)\n" + - " │ ├─ index: [rs.s]\n" + - " │ ├─ filters: [{[NULL, ∞)}]\n" + + " └─ LeftOuterLookupJoin\n" + + " ├─ Table\n" + + " │ ├─ name: rs\n" + " │ └─ columns: [r s]\n" + " └─ IndexedTableAccess(xy)\n" + " ├─ index: [xy.y]\n" + - " ├─ filters: [{[NULL, ∞)}]\n" + - " └─ columns: [x y]\n" + + " ├─ columns: [x y]\n" + + " └─ keys: rs.s\n" + "", ExpectedAnalysis: "Project\n" + " ├─ columns: [rs.r, rs.s, xy.x, xy.y]\n" + " └─ Sort(rs.r ASC, xy.x ASC)\n" + - " └─ LeftOuterMergeJoin\n" + - " ├─ cmp: (rs.s = xy.y)\n" + - " ├─ IndexedTableAccess(rs)\n" + - " │ ├─ index: [rs.s]\n" + - " │ ├─ filters: [{[NULL, ∞)}]\n" + + " └─ LeftOuterLookupJoin\n" + + " ├─ Table\n" + + " │ ├─ name: rs\n" + " │ └─ columns: [r s]\n" + " └─ IndexedTableAccess(xy)\n" + " ├─ index: [xy.y]\n" + - " ├─ filters: [{[NULL, ∞)}]\n" + - " └─ columns: [x y]\n" + + " ├─ columns: [x y]\n" + + " └─ keys: rs.s\n" + "", }, { @@ -6081,23 +6091,17 @@ inner join pq on true " │ │ ├─ columns: [ab.a:0!null, ab.b:1]\n" + " │ │ └─ Filter\n" + " │ │ ├─ xy.x:2!null IS NULL\n" + - " │ │ └─ LeftOuterMergeJoin\n" + - " │ │ ├─ cmp: Eq\n" + - " │ │ │ ├─ ab.a:0!null\n" + - " │ │ │ └─ xy.x:2!null\n" + - " │ │ ├─ IndexedTableAccess(ab)\n" + - " │ │ │ ├─ index: [ab.a]\n" + - " │ │ │ ├─ static: [{[NULL, ∞)}]\n" + + " │ │ └─ LeftOuterLookupJoin\n" + + " │ │ ├─ Table\n" + + " │ │ │ ├─ name: ab\n" + + " │ │ │ ├─ columns: [a b]\n" + " │ │ │ ├─ colSet: (1,2)\n" + - " │ │ │ ├─ tableId: 1\n" + - " │ │ │ └─ Table\n" + - " │ │ │ ├─ name: ab\n" + - " │ │ │ └─ columns: [a b]\n" + + " │ │ │ └─ tableId: 1\n" + " │ │ └─ Project\n" + " │ │ ├─ columns: [xy.x:0!null]\n" + " │ │ └─ IndexedTableAccess(xy)\n" + " │ │ ├─ index: [xy.x]\n" + - " │ │ ├─ static: [{[NULL, ∞)}]\n" + + " │ │ ├─ keys: [ab.a:0!null]\n" + " │ │ ├─ colSet: (3,4)\n" + " │ │ ├─ tableId: 2\n" + " │ │ └─ Table\n" + @@ -6130,17 +6134,15 @@ inner join pq on true " │ │ ├─ columns: [ab.a, ab.b]\n" + " │ │ └─ Filter\n" + " │ │ ├─ xy.x IS NULL\n" + - " │ │ └─ LeftOuterMergeJoin\n" + - " │ │ ├─ cmp: (ab.a = xy.x)\n" + - " │ │ ├─ IndexedTableAccess(ab)\n" + - " │ │ │ ├─ index: [ab.a]\n" + - " │ │ │ └─ filters: [{[NULL, ∞)}]\n" + + " │ │ └─ LeftOuterLookupJoin\n" + + " │ │ ├─ Table\n" + + " │ │ │ └─ name: ab\n" + " │ │ └─ Project\n" + " │ │ ├─ columns: [xy.x]\n" + " │ │ └─ IndexedTableAccess(xy)\n" + " │ │ ├─ index: [xy.x]\n" + - " │ │ ├─ filters: [{[NULL, ∞)}]\n" + - " │ │ └─ columns: [x y]\n" + + " │ │ ├─ columns: [x y]\n" + + " │ │ └─ keys: ab.a\n" + " │ └─ HashLookup\n" + " │ ├─ left-key: (alias1.a)\n" + " │ ├─ right-key: (pq.p)\n" + @@ -6165,17 +6167,15 @@ inner join pq on true " │ │ ├─ columns: [ab.a, ab.b]\n" + " │ │ └─ Filter\n" + " │ │ ├─ xy.x IS NULL\n" + - " │ │ └─ LeftOuterMergeJoin\n" + - " │ │ ├─ cmp: (ab.a = xy.x)\n" + - " │ │ ├─ IndexedTableAccess(ab)\n" + - " │ │ │ ├─ index: [ab.a]\n" + - " │ │ │ └─ filters: [{[NULL, ∞)}]\n" + + " │ │ └─ LeftOuterLookupJoin\n" + + " │ │ ├─ Table\n" + + " │ │ │ └─ name: ab\n" + " │ │ └─ Project\n" + " │ │ ├─ columns: [xy.x]\n" + " │ │ └─ IndexedTableAccess(xy)\n" + " │ │ ├─ index: [xy.x]\n" + - " │ │ ├─ filters: [{[NULL, ∞)}]\n" + - " │ │ └─ columns: [x y]\n" + + " │ │ ├─ columns: [x y]\n" + + " │ │ └─ keys: ab.a\n" + " │ └─ HashLookup\n" + " │ ├─ left-key: (alias1.a)\n" + " │ ├─ right-key: (pq.p)\n" + @@ -14434,49 +14434,51 @@ inner join pq on true { Query: `SELECT l.i, r.i2 FROM niltable l INNER JOIN niltable r ON l.i2 <=> r.i2 ORDER BY 1 ASC`, ExpectedPlan: "Project\n" + - " ├─ columns: [l.i:1!null, r.i2:0]\n" + - " └─ Sort(l.i:1!null ASC nullsFirst)\n" + - " └─ InnerJoin\n" + - " ├─ (l.i2:2 <=> r.i2:0)\n" + - " ├─ TableAlias(r)\n" + + " ├─ columns: [l.i:0!null, r.i2:2]\n" + + " └─ Sort(l.i:0!null ASC nullsFirst)\n" + + " └─ LookupJoin\n" + + " ├─ TableAlias(l)\n" + " │ └─ ProcessTable\n" + " │ └─ Table\n" + " │ ├─ name: niltable\n" + - " │ └─ columns: [i2]\n" + - " └─ TableAlias(l)\n" + - " └─ Table\n" + - " ├─ name: niltable\n" + - " ├─ columns: [i i2]\n" + - " ├─ colSet: (1-4)\n" + - " └─ tableId: 1\n" + + " │ └─ columns: [i i2]\n" + + " └─ TableAlias(r)\n" + + " └─ IndexedTableAccess(niltable)\n" + + " ├─ index: [niltable.i2]\n" + + " ├─ keys: [l.i2:1]\n" + + " ├─ colSet: (5-8)\n" + + " ├─ tableId: 2\n" + + " └─ Table\n" + + " ├─ name: niltable\n" + + " └─ columns: [i2]\n" + "", ExpectedEstimates: "Project\n" + " ├─ columns: [l.i, r.i2]\n" + " └─ Sort(l.i ASC)\n" + - " └─ InnerJoin\n" + - " ├─ (l.i2 <=> r.i2)\n" + - " ├─ TableAlias(r)\n" + + " └─ LookupJoin\n" + + " ├─ TableAlias(l)\n" + " │ └─ Table\n" + " │ ├─ name: niltable\n" + - " │ └─ columns: [i2]\n" + - " └─ TableAlias(l)\n" + - " └─ Table\n" + - " ├─ name: niltable\n" + - " └─ columns: [i i2]\n" + + " │ └─ columns: [i i2]\n" + + " └─ TableAlias(r)\n" + + " └─ IndexedTableAccess(niltable)\n" + + " ├─ index: [niltable.i2]\n" + + " ├─ columns: [i2]\n" + + " └─ keys: l.i2\n" + "", ExpectedAnalysis: "Project\n" + " ├─ columns: [l.i, r.i2]\n" + " └─ Sort(l.i ASC)\n" + - " └─ InnerJoin\n" + - " ├─ (l.i2 <=> r.i2)\n" + - " ├─ TableAlias(r)\n" + + " └─ LookupJoin\n" + + " ├─ TableAlias(l)\n" + " │ └─ Table\n" + " │ ├─ name: niltable\n" + - " │ └─ columns: [i2]\n" + - " └─ TableAlias(l)\n" + - " └─ Table\n" + - " ├─ name: niltable\n" + - " └─ columns: [i i2]\n" + + " │ └─ columns: [i i2]\n" + + " └─ TableAlias(r)\n" + + " └─ IndexedTableAccess(niltable)\n" + + " ├─ index: [niltable.i2]\n" + + " ├─ columns: [i2]\n" + + " └─ keys: l.i2\n" + "", }, { @@ -25111,7 +25113,7 @@ order by x, y; " │ ├─ name: xy\n" + " │ └─ columns: [x y]\n" + " └─ IndexedTableAccess(mytable)\n" + - " ├─ index: [mytable.s]\n" + + " ├─ index: [mytable.s,mytable.i]\n" + " ├─ keys: [xy.x:0!null]\n" + " ├─ colSet: (3,4)\n" + " ├─ tableId: 2\n" + @@ -25119,21 +25121,21 @@ order by x, y; " ├─ name: mytable\n" + " └─ columns: [i s]\n" + "", - ExpectedEstimates: "LookupJoin (estimated cost=1006.900 rows=3)\n" + + ExpectedEstimates: "LookupJoin (estimated cost=2023.000 rows=3)\n" + " ├─ Table\n" + " │ ├─ name: xy\n" + " │ └─ columns: [x y]\n" + " └─ IndexedTableAccess(mytable)\n" + - " ├─ index: [mytable.s]\n" + + " ├─ index: [mytable.s,mytable.i]\n" + " ├─ columns: [i s]\n" + " └─ keys: xy.x\n" + "", - ExpectedAnalysis: "LookupJoin (estimated cost=1006.900 rows=3) (actual rows=0 loops=1)\n" + + ExpectedAnalysis: "LookupJoin (estimated cost=2023.000 rows=3) (actual rows=0 loops=1)\n" + " ├─ Table\n" + " │ ├─ name: xy\n" + " │ └─ columns: [x y]\n" + " └─ IndexedTableAccess(mytable)\n" + - " ├─ index: [mytable.s]\n" + + " ├─ index: [mytable.s,mytable.i]\n" + " ├─ columns: [i s]\n" + " └─ keys: xy.x\n" + "", diff --git a/enginetest/queries/tpcc_plans.go b/enginetest/queries/tpcc_plans.go index fa2de7451d..fa343c5d8a 100644 --- a/enginetest/queries/tpcc_plans.go +++ b/enginetest/queries/tpcc_plans.go @@ -709,8 +709,8 @@ SELECT d_next_o_id FROM district2 WHERE d_id = 5 AND d_w_id= 1`, " │ ├─ stock2.s_quantity:2\n" + " │ └─ 18 (smallint)\n" + " └─ IndexedTableAccess(stock2)\n" + - " ├─ index: [stock2.s_w_id,stock2.s_i_id]\n" + - " ├─ keys: [1 (smallint) order_line2.ol_i_id:3]\n" + + " ├─ index: [stock2.s_i_id]\n" + + " ├─ keys: [order_line2.ol_i_id:3]\n" + " ├─ colSet: (11-27)\n" + " ├─ tableId: 2\n" + " └─ Table\n" + @@ -730,9 +730,9 @@ SELECT d_next_o_id FROM district2 WHERE d_id = 5 AND d_w_id= 1`, " └─ Filter\n" + " ├─ ((stock2.s_w_id = 1) AND (stock2.s_quantity < 18))\n" + " └─ IndexedTableAccess(stock2)\n" + - " ├─ index: [stock2.s_w_id,stock2.s_i_id]\n" + + " ├─ index: [stock2.s_i_id]\n" + " ├─ columns: [s_i_id s_w_id s_quantity]\n" + - " └─ keys: 1, order_line2.ol_i_id\n" + + " └─ keys: order_line2.ol_i_id\n" + "", ExpectedAnalysis: "Project\n" + " ├─ columns: [countdistinct([stock2.s_i_id]) as COUNT(DISTINCT (s_i_id))]\n" + @@ -747,9 +747,9 @@ SELECT d_next_o_id FROM district2 WHERE d_id = 5 AND d_w_id= 1`, " └─ Filter\n" + " ├─ ((stock2.s_w_id = 1) AND (stock2.s_quantity < 18))\n" + " └─ IndexedTableAccess(stock2)\n" + - " ├─ index: [stock2.s_w_id,stock2.s_i_id]\n" + + " ├─ index: [stock2.s_i_id]\n" + " ├─ columns: [s_i_id s_w_id s_quantity]\n" + - " └─ keys: 1, order_line2.ol_i_id\n" + + " └─ keys: order_line2.ol_i_id\n" + "", }, { diff --git a/memory/table.go b/memory/table.go index bde14a122f..68fd13664e 100644 --- a/memory/table.go +++ b/memory/table.go @@ -508,6 +508,10 @@ func (i *indexScanRowIter) Next(ctx *sql.Context) (sql.Row, error) { } func indexRowMatches(ranges sql.Expression, candidate sql.Row) (bool, error) { + if ranges == nil { + // If there is no ranges expression, default to match all (return all rows) + return true, nil + } result, err := ranges.Eval(nil, candidate) if err != nil { return false, err @@ -1685,6 +1689,10 @@ func (t *IndexedTable) LookupPartitions(ctx *sql.Context, lookup sql.IndexLookup } func adjustRangeScanFilterForIndexLookup(filter sql.Expression, index *Index) sql.Expression { + if filter == nil { + return filter + } + exprs := index.ExtendedExprs() indexStorageSchema := make(sql.Schema, len(exprs)) diff --git a/sql/memo/coster.go b/sql/memo/coster.go index a313f6aa00..f5aba01678 100644 --- a/sql/memo/coster.go +++ b/sql/memo/coster.go @@ -94,6 +94,7 @@ func (c *coster) costRel(ctx *sql.Context, n RelExpr, s sql.StatsProvider) (floa case jp.Op.IsDegenerate(): return ((lBest*rBest)*seqIOCostFactor + (lBest*rBest)*cpuCostFactor) * degeneratePenalty, nil case jp.Op.IsHash(): + // TODO we're likely underestimating the cost of hash joins because we don't account for hash setup costs if jp.Op.IsPartial() { cost := lBest * (rBest / 2.0) * (seqIOCostFactor + cpuCostFactor) return cost * .5, nil @@ -121,18 +122,25 @@ func (c *coster) costRel(ctx *sql.Context, n RelExpr, s sql.StatsProvider) (floa // TODO: estimate memory overhead return float64(lTableScan+rTableScan)*(seqIOCostFactor+cpuCostFactor) + cpuCostFactor*selfJoinCard, nil case jp.Op.IsLookup(): - // TODO added overhead for right lookups switch n := n.(type) { case *LookupJoin: - if !n.Injective { - // partial index completion is undesirable - // TODO don't do this whe we have stats - selfJoinCard = math.Max(0, selfJoinCard+float64(indexCoverageAdjustment(n.Lookup))) + // Match rate is what proportion of left-side rows expected to match at least one row on the right + matchRate := lookupJoinMatchRate(n.Lookup, n.JoinBase) + + // If LookupJoin is injective, then there will only be one right lookup per left row + if n.Injective || matchRate == 0 { + return lBest*seqIOCostFactor + lBest*(cpuCostFactor+randIOCostFactor), nil } - // read the whole left table and randIO into table equivalent to - // this join's output cardinality estimate - return lBest*seqIOCostFactor + selfJoinCard*(randIOCostFactor+seqIOCostFactor), nil + // The total expected number of right row lookups + expectedRightRows := selfJoinCard * matchRate + + if expectedRightRows < lBest { + return lBest*(seqIOCostFactor) + (lBest+indexCoverageAdjustment(n.Lookup))*(cpuCostFactor+randIOCostFactor), nil + } + + // Estimate for reading each left row and each expected right row + return lBest*seqIOCostFactor + expectedRightRows*(cpuCostFactor+randIOCostFactor), nil case *ConcatJoin: return c.costConcatJoin(ctx, n, s) } @@ -228,15 +236,17 @@ func (c *coster) costConcatJoin(_ *sql.Context, n *ConcatJoin, _ sql.StatsProvid var sel float64 for _, l := range n.Concat { lookup := l - sel += lookupJoinSelectivity(lookup, n.JoinBase) + sel += lookupJoinMatchRate(lookup, n.JoinBase) } return l*sel*concatCostFactor*(randIOCostFactor+cpuCostFactor) - float64(n.Right.RelProps.GetStats().RowCount())*seqIOCostFactor, nil } -// lookupJoinSelectivity estimates the selectivity of a join condition with n lhs rows and m rhs rows. -// A join with a selectivity of k will return k*(n*m) rows. -// Special case: A join with a selectivity of 0 will return n rows. -func lookupJoinSelectivity(l *IndexScan, joinBase *JoinBase) float64 { +// lookupJoinMatchRate returns a heuristic estimate of the proportion of right-side rows matched +// for each left-side row in a LookupJoin. Lower values indicate higher selectivity (i.e., more filtering). +// +// Special case: If the lookup is injective (i.e., at most one match per left row), we return 0 to +// indicate that join cardinality is ≤ the left-side cardinality. +func lookupJoinMatchRate(l *IndexScan, joinBase *JoinBase) float64 { if isInjectiveLookup(l.Index, joinBase, l.Table.Expressions(), l.Table.NullMask()) { return 0 }