Skip to content

Commit 07e1501

Browse files
mgartnermgartner
committed
opt: reduce regression in join reordering allocations
PR #151504 fixed a bug but caused a regression in allocations, particularly for `slow-query-1`. This commit reduces the regression by altering methods which extract not-null columns from filters and constraints—they now add columns to existing sets instead of creating new sets. Release note: None
1 parent 5cb36ef commit 07e1501

File tree

9 files changed

+53
-51
lines changed

9 files changed

+53
-51
lines changed

‎pkg/sql/opt/constraint/constraint.go‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -725,15 +725,15 @@ func (c *Constraint) ExtractConstCols(ctx context.Context, evalCtx *eval.Context
725725
return res
726726
}
727727

728-
// ExtractNotNullCols returns a set of columns that cannot be NULL when the
729-
// constraint holds.
730-
func (c *Constraint) ExtractNotNullCols(ctx context.Context, evalCtx *eval.Context) opt.ColSet {
728+
// ExtractNotNullCols finds the set of columns that cannot be NULL without
729+
// violating the constraint, and adds them to the given column set.
730+
func (c *Constraint) ExtractNotNullCols(
731+
ctx context.Context, evalCtx *eval.Context, cols *opt.ColSet,
732+
) {
731733
if c.IsUnconstrained() || c.IsContradiction() {
732-
return opt.ColSet{}
734+
return
733735
}
734736

735-
var res opt.ColSet
736-
737737
// If we have a span where the start and end key value diverge for a column,
738738
// none of the columns that follow can be not-null. For example:
739739
// /1/2/3: [/1/2/3 - /1/4/1]
@@ -754,11 +754,11 @@ func (c *Constraint) ExtractNotNullCols(ctx context.Context, evalCtx *eval.Conte
754754
hasNull = hasNull || start.Value(i) == tree.DNull
755755
}
756756
if !hasNull {
757-
res.Add(c.Columns.Get(i).ID())
757+
cols.Add(c.Columns.Get(i).ID())
758758
}
759759
}
760760
if prefix == c.Columns.Count() {
761-
return res
761+
return
762762
}
763763

764764
// Now look at the first column that follows the prefix.
@@ -775,12 +775,11 @@ func (c *Constraint) ExtractNotNullCols(ctx context.Context, evalCtx *eval.Conte
775775
// If the span is unbounded on the NULL side, or if it is of the form
776776
// [/NULL - /x], the column is nullable.
777777
if key.Length() <= prefix || (key.Value(prefix) == tree.DNull && boundary == IncludeBoundary) {
778-
return res
778+
return
779779
}
780780
}
781781
// All spans constrain col to be not-null.
782-
res.Add(col.ID())
783-
return res
782+
cols.Add(col.ID())
784783
}
785784

786785
// CalculateMaxResults returns an integer indicating the maximum number of
@@ -803,7 +802,9 @@ func (c *Constraint) CalculateMaxResults(
803802
// Ensure that if we have nullable columns, we are only reading non-null
804803
// values, given that a unique index allows an arbitrary number of duplicate
805804
// entries if they have NULLs.
806-
if !indexCols.SubsetOf(notNullCols.Union(c.ExtractNotNullCols(ctx, evalCtx))) {
805+
notNullCols = notNullCols.Copy()
806+
c.ExtractNotNullCols(ctx, evalCtx, &notNullCols)
807+
if !indexCols.SubsetOf(notNullCols) {
807808
return 0, false
808809
}
809810

‎pkg/sql/opt/constraint/constraint_set.go‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -262,17 +262,16 @@ func (s *Set) ExtractCols() opt.ColSet {
262262
return res
263263
}
264264

265-
// ExtractNotNullCols returns a set of columns that cannot be NULL for the
266-
// constraints in the set to hold.
267-
func (s *Set) ExtractNotNullCols(ctx context.Context, evalCtx *eval.Context) opt.ColSet {
265+
// ExtractNotNullCols finds the set of columns that cannot be NULL without
266+
// violating the constraints in the set, and adds them to the given column set.
267+
func (s *Set) ExtractNotNullCols(ctx context.Context, evalCtx *eval.Context, cols *opt.ColSet) {
268268
if s == Unconstrained || s == Contradiction {
269-
return opt.ColSet{}
269+
return
270270
}
271-
res := s.Constraint(0).ExtractNotNullCols(ctx, evalCtx)
271+
s.Constraint(0).ExtractNotNullCols(ctx, evalCtx, cols)
272272
for i := 1; i < s.Length(); i++ {
273-
res.UnionWith(s.Constraint(i).ExtractNotNullCols(ctx, evalCtx))
273+
s.Constraint(i).ExtractNotNullCols(ctx, evalCtx, cols)
274274
}
275-
return res
276275
}
277276

278277
// ExtractConstCols returns a set of columns which can only have one value

‎pkg/sql/opt/constraint/constraint_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1229,7 +1229,8 @@ func TestExtractNotNullCols(t *testing.T) {
12291229
for i, tc := range testData {
12301230
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
12311231
c := ParseConstraint(&evalCtx, tc.c)
1232-
cols := c.ExtractNotNullCols(ctx, &evalCtx)
1232+
var cols opt.ColSet
1233+
c.ExtractNotNullCols(ctx, &evalCtx, &cols)
12331234
if exp := opt.MakeColSet(tc.e...); !cols.Equals(exp) {
12341235
t.Errorf("expected %s; got %s", exp, cols)
12351236
}

‎pkg/sql/opt/memo/logical_props_builder.go‎

Lines changed: 20 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -93,12 +93,12 @@ func (b *logicalPropsBuilder) buildScanProps(scan *ScanExpr, rel *props.Relation
9393
rel.NotNullCols = makeTableNotNullCols(md, scan.Table).Copy()
9494
// Union not-NULL columns with not-NULL columns in the constraint.
9595
if scan.Constraint != nil {
96-
rel.NotNullCols.UnionWith(scan.Constraint.ExtractNotNullCols(b.sb.ctx, b.evalCtx))
96+
scan.Constraint.ExtractNotNullCols(b.sb.ctx, b.evalCtx, &rel.NotNullCols)
9797
}
9898
// Union not-NULL columns with not-NULL columns in the partial index
9999
// predicate.
100100
if pred != nil {
101-
rel.NotNullCols.UnionWith(b.rejectNullCols(pred))
101+
b.extractNotNullCols(pred, &rel.NotNullCols)
102102
}
103103
rel.NotNullCols.IntersectionWith(rel.OutputCols)
104104

@@ -267,7 +267,8 @@ func (b *logicalPropsBuilder) buildSelectProps(sel *SelectExpr, rel *props.Relat
267267
// SELECT y FROM xy WHERE y=5
268268
//
269269
// "y" cannot be null because the SQL equality operator rejects nulls.
270-
rel.NotNullCols = b.rejectNullCols(sel.Filters)
270+
rel.NotNullCols = opt.ColSet{}
271+
b.extractNotNullCols(sel.Filters, &rel.NotNullCols)
271272
rel.NotNullCols.UnionWith(inputProps.NotNullCols)
272273
rel.NotNullCols.IntersectionWith(rel.OutputCols)
273274

@@ -2215,26 +2216,24 @@ func (b *logicalPropsBuilder) makeSetCardinality(
22152216
return card
22162217
}
22172218

2218-
// NullColsRejectedByFilter returns a set of columns that are "null rejected"
2219-
// by the filters. An input row with a NULL value on any of these columns will
2220-
// not pass the filter.
2221-
func NullColsRejectedByFilter(
2222-
ctx context.Context, evalCtx *eval.Context, filters FiltersExpr,
2223-
) opt.ColSet {
2224-
var notNullCols opt.ColSet
2219+
// ExtractNullColsRejectedByFilter finds the set of columns that are "null
2220+
// rejected" by the filters and adds them to the given column set. An input row
2221+
// with a NULL value on any of these columns will not pass the filter.
2222+
func ExtractNullColsRejectedByFilter(
2223+
ctx context.Context, evalCtx *eval.Context, filters FiltersExpr, cols *opt.ColSet,
2224+
) {
22252225
for i := range filters {
22262226
filterProps := filters[i].ScalarProps()
22272227
if filterProps.Constraints != nil {
2228-
notNullCols.UnionWith(filterProps.Constraints.ExtractNotNullCols(ctx, evalCtx))
2228+
filterProps.Constraints.ExtractNotNullCols(ctx, evalCtx, cols)
22292229
}
22302230
}
2231-
return notNullCols
22322231
}
22332232

2234-
// rejectNullCols returns the set of all columns that are inferred to be not-
2235-
// null, based on the filter conditions.
2236-
func (b *logicalPropsBuilder) rejectNullCols(filters FiltersExpr) opt.ColSet {
2237-
return NullColsRejectedByFilter(b.sb.ctx, b.evalCtx, filters)
2233+
// extractNotNullCols finds the set of columns that are "null rejected" by the
2234+
// filters and adds them to the given column set.
2235+
func (b *logicalPropsBuilder) extractNotNullCols(filters FiltersExpr, cols *opt.ColSet) {
2236+
ExtractNullColsRejectedByFilter(b.sb.ctx, b.evalCtx, filters, cols)
22382237
}
22392238

22402239
// addFiltersToFuncDep returns the union of all functional dependencies from
@@ -2548,7 +2547,7 @@ func (h *joinPropsHelper) init(b *logicalPropsBuilder, joinExpr RelExpr) {
25482547
h.rightProps = &join.lookupProps
25492548
h.filters = append(join.On, join.AllLookupFilters...)
25502549
b.addFiltersToFuncDep(h.filters, &h.filtersFD)
2551-
h.filterNotNullCols = b.rejectNullCols(h.filters)
2550+
b.extractNotNullCols(h.filters, &h.filterNotNullCols)
25522551

25532552
// Apply the lookup join equalities.
25542553
index := md.Table(join.Table).Index(join.Index)
@@ -2574,7 +2573,7 @@ func (h *joinPropsHelper) init(b *logicalPropsBuilder, joinExpr RelExpr) {
25742573
h.rightProps = &join.lookupProps
25752574
h.filters = join.On
25762575
b.addFiltersToFuncDep(h.filters, &h.filtersFD)
2577-
h.filterNotNullCols = b.rejectNullCols(h.filters)
2576+
b.extractNotNullCols(h.filters, &h.filterNotNullCols)
25782577

25792578
// Apply the prefix column equalities.
25802579
index := md.Table(join.Table).Index(join.Index)
@@ -2599,7 +2598,7 @@ func (h *joinPropsHelper) init(b *logicalPropsBuilder, joinExpr RelExpr) {
25992598
h.rightProps = join.Right.Relational()
26002599
h.filters = join.On
26012600
b.addFiltersToFuncDep(h.filters, &h.filtersFD)
2602-
h.filterNotNullCols = b.rejectNullCols(h.filters)
2601+
b.extractNotNullCols(h.filters, &h.filterNotNullCols)
26032602

26042603
// Apply the merge join equalities.
26052604
for i := range join.LeftEq {
@@ -2621,7 +2620,7 @@ func (h *joinPropsHelper) init(b *logicalPropsBuilder, joinExpr RelExpr) {
26212620
h.rightProps = &join.rightProps
26222621
h.filters = join.On
26232622
b.addFiltersToFuncDep(h.filters, &h.filtersFD)
2624-
h.filterNotNullCols = b.rejectNullCols(h.filters)
2623+
b.extractNotNullCols(h.filters, &h.filterNotNullCols)
26252624

26262625
// Apply the zigzag join equalities.
26272626
for i := range join.LeftEqCols {
@@ -2640,7 +2639,7 @@ func (h *joinPropsHelper) init(b *logicalPropsBuilder, joinExpr RelExpr) {
26402639

26412640
h.filters = *join.Child(2).(*FiltersExpr)
26422641
b.addFiltersToFuncDep(h.filters, &h.filtersFD)
2643-
h.filterNotNullCols = b.rejectNullCols(h.filters)
2642+
b.extractNotNullCols(h.filters, &h.filterNotNullCols)
26442643
h.filterIsTrue = h.filters.IsTrue()
26452644
h.filterIsFalse = h.filters.IsFalse()
26462645
}

‎pkg/sql/opt/memo/statistics_builder.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -872,7 +872,7 @@ func (sb *statisticsBuilder) buildScan(scan *ScanExpr, relProps *props.Relationa
872872
// Add any not-null columns from the predicate constraints.
873873
for i := range pred {
874874
if c := pred[i].ScalarProps().Constraints; c != nil {
875-
notNullCols.UnionWith(c.ExtractNotNullCols(sb.ctx, sb.evalCtx))
875+
c.ExtractNotNullCols(sb.ctx, sb.evalCtx, &notNullCols)
876876
}
877877
}
878878
sb.filterRelExpr(pred, scan, notNullCols, relProps, s, MakeTableFuncDep(sb.md, scan.Table))
@@ -1046,12 +1046,12 @@ func (sb *statisticsBuilder) constrainScan(
10461046
notNullCols := relProps.NotNullCols.Copy()
10471047
if constraint != nil {
10481048
// Add any not-null columns from this constraint.
1049-
notNullCols.UnionWith(constraint.ExtractNotNullCols(sb.ctx, sb.evalCtx))
1049+
constraint.ExtractNotNullCols(sb.ctx, sb.evalCtx, &notNullCols)
10501050
}
10511051
// Add any not-null columns from the predicate constraints.
10521052
for i := range pred {
10531053
if c := pred[i].ScalarProps().Constraints; c != nil {
1054-
notNullCols.UnionWith(c.ExtractNotNullCols(sb.ctx, sb.evalCtx))
1054+
c.ExtractNotNullCols(sb.ctx, sb.evalCtx, &notNullCols)
10551055
}
10561056
}
10571057
sb.updateNullCountsFromNotNullCols(notNullCols, s)
@@ -3710,7 +3710,7 @@ func (sb *statisticsBuilder) constrainExpr(
37103710
// ---------------------------------------------
37113711
notNullCols := relProps.NotNullCols.Copy()
37123712
// Add any not-null columns from this constraint set.
3713-
notNullCols.UnionWith(cs.ExtractNotNullCols(sb.ctx, sb.evalCtx))
3713+
cs.ExtractNotNullCols(sb.ctx, sb.evalCtx, &notNullCols)
37143714
sb.updateNullCountsFromNotNullCols(notNullCols, s)
37153715

37163716
// Calculate row count and selectivity

‎pkg/sql/opt/memo/statistics_builder_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ func TestGetStatsFromConstraint(t *testing.T) {
110110
sel := mem.MemoizeSelect(scan, TrueFilter)
111111

112112
relProps := &props.Relational{Cardinality: props.AnyCardinality}
113-
relProps.NotNullCols = cs.ExtractNotNullCols(ctx, &evalCtx)
113+
cs.ExtractNotNullCols(ctx, &evalCtx, &relProps.NotNullCols)
114114
s := relProps.Statistics()
115115
const minRowCount = 0
116116
s.Init(relProps, minRowCount)

‎pkg/sql/opt/norm/reject_nulls_funcs.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ func (c *CustomFuncs) HasNullRejectingFilter(
3535
continue
3636
}
3737

38-
notNullFilterCols := constraints.ExtractNotNullCols(c.f.ctx, c.f.evalCtx)
38+
var notNullFilterCols opt.ColSet
39+
constraints.ExtractNotNullCols(c.f.ctx, c.f.evalCtx, &notNullFilterCols)
3940
if notNullFilterCols.Intersects(nullRejectCols) {
4041
return true
4142
}
@@ -286,7 +287,7 @@ func (c *CustomFuncs) GetNullRejectedCols(filters memo.FiltersExpr) opt.ColSet {
286287
continue
287288
}
288289

289-
nullRejectedCols.UnionWith(constraints.ExtractNotNullCols(c.f.ctx, c.f.evalCtx))
290+
constraints.ExtractNotNullCols(c.f.ctx, c.f.evalCtx, &nullRejectedCols)
290291
}
291292
return nullRejectedCols
292293
}

‎pkg/sql/opt/xform/join_funcs.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1221,7 +1221,8 @@ func (c *CustomFuncs) FindLeftJoinCanaryColumn(
12211221
// Find any column from the right which is null-rejected by the ON condition.
12221222
// right rows where such a column is NULL will never contribute to the join
12231223
// result.
1224-
nullRejectedCols := memo.NullColsRejectedByFilter(c.e.ctx, c.e.evalCtx, on)
1224+
var nullRejectedCols opt.ColSet
1225+
memo.ExtractNullColsRejectedByFilter(c.e.ctx, c.e.evalCtx, on, &nullRejectedCols)
12251226
nullRejectedCols.IntersectionWith(right.Relational().OutputCols)
12261227

12271228
canaryCol, ok = nullRejectedCols.Next(0)

‎pkg/sql/opt/xform/join_order_builder.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -642,7 +642,7 @@ func (jb *JoinOrderBuilder) addJoins(s1, s2 vertexSet) {
642642
redundant := areFiltersRedundant(&jb.equivs, e.filters, notNullCols)
643643
// Update notNullCols based on null-rejecting filters to aid in
644644
// finding subsequent redundant filters.
645-
notNullCols.UnionWith(memo.NullColsRejectedByFilter(jb.ctx, jb.evalCtx, e.filters))
645+
memo.ExtractNullColsRejectedByFilter(jb.ctx, jb.evalCtx, e.filters, &notNullCols)
646646
if redundant {
647647
// Avoid adding redundant filters.
648648
continue
@@ -1256,7 +1256,7 @@ func (e *edge) calcNullRejectedRels(jb *JoinOrderBuilder) {
12561256
var nullRejectedCols opt.ColSet
12571257
for i := range e.filters {
12581258
if constraints := e.filters[i].ScalarProps().Constraints; constraints != nil {
1259-
nullRejectedCols.UnionWith(constraints.ExtractNotNullCols(jb.ctx, jb.evalCtx))
1259+
constraints.ExtractNotNullCols(jb.ctx, jb.evalCtx, &nullRejectedCols)
12601260
}
12611261
}
12621262
e.nullRejectedRels = jb.getRelations(nullRejectedCols)

0 commit comments

Comments
 (0)