opt: make CostFlags an ordered bitmask

`memo.CostFlags` is now an ordered bitmask where each bit indicates a
specific cost penalty and the penalties are ordered by precedence. The
highest precedence penalty uses the highest-order bit. This allows
CostFlags to be easily compared with built-in comparison operators (>,
<, =).

Release note: None

Author: mgartner

pkg/sql/opt/memo/cost.go

@@ -27,18 +27,14 @@ type Cost struct {
 // group members during testing, by setting their cost so high that any other
 // member will have a lower cost.
 var MaxCost = Cost{
-	C: math.Inf(+1),
-	Flags: CostFlags{
-		FullScanPenalty:      true,
-		HugeCostPenalty:      true,
-		UnboundedCardinality: true,
-	},
+	C:     math.Inf(+1),
+	Flags: HugeCostPenalty | FullScanPenalty | UnboundedCardinality,
 }
 
 // Less returns true if this cost is lower than the given cost.
 func (c Cost) Less(other Cost) bool {
 	if c.Flags != other.Flags {
-		return c.Flags.Less(other.Flags)
+		return c.Flags < other.Flags
 	}
 	// Two plans with the same cost can have slightly different floating point
 	// results (e.g. same subcosts being added up in a different order). So we
@@ -56,7 +52,7 @@ func (c Cost) Less(other Cost) bool {
 // Add adds the other cost to this cost.
 func (c *Cost) Add(other Cost) {
 	c.C += other.C
-	c.Flags.Add(other.Flags)
+	c.Flags |= other.Flags
 	if c.aux.fullScanCount > math.MaxUint8-other.aux.fullScanCount {
 		// Avoid overflow.
 		c.aux.fullScanCount = math.MaxUint8
@@ -79,48 +75,31 @@ func (c *Cost) IncrFullScanCount() {
 	c.aux.fullScanCount++
 }
 
-// CostFlags contains flags that penalize the cost of an operator.
-type CostFlags struct {
-	// FullScanPenalty is true if the cost of a full table or index scan is
-	// penalized, indicating that a full scan should only be used if no other plan
-	// is possible.
-	FullScanPenalty bool
+// CostFlags is an ordered bitmask where each bit indicates a cost penalty. The
+// penalties are ordered by precedence, with the highest precedence penalty
+// using the highest-order bit. This allows CostFlags to be easily compared with
+// built-in comparison operators (>, <, =, etc.). For example, CostFlags with
+// HugeCostPenalty will always be greater than CostFlags without.
+type CostFlags uint8
+
+const (
 	// HugeCostPenalty is true if a plan should be avoided at all costs. This is
-	// used when the optimizer is forced to use a particular plan, and will error
-	// if it cannot be used.
-	HugeCostPenalty bool
+	// used when the optimizer is forced to use a particular plan, and will
+	// error if it cannot be used. It takes precedence over other penalties,
+	// since it indicates that a plan is being forced with a hint, and will
+	// error if we cannot comply with the hint.
+	HugeCostPenalty CostFlags = 1 << (7 - iota)
+
+	// FullScanPenalty is true if the cost of a full table or index scan is
+	// penalized, indicating that a full scan should only be used if no other
+	// plan is possible.
+	FullScanPenalty
+
 	// UnboundedCardinality is true if the operator or any of its descendants
 	// have no guaranteed upperbound on the number of rows that they can
 	// produce. See props.AnyCardinality.
-	UnboundedCardinality bool
-}
-
-// Less returns true if these flags indicate a lower penalty than the other
-// CostFlags.
-func (c CostFlags) Less(other CostFlags) bool {
-	// HugeCostPenalty takes precedence over other penalties, since it indicates
-	// that a plan is being forced with a hint, and will error if we cannot comply
-	// with the hint.
-	if c.HugeCostPenalty != other.HugeCostPenalty {
-		return !c.HugeCostPenalty
-	}
-	if c.FullScanPenalty != other.FullScanPenalty {
-		return !c.FullScanPenalty
-	}
-	if c.UnboundedCardinality != other.UnboundedCardinality {
-		return !c.UnboundedCardinality
-	}
-	return false
-}
+	UnboundedCardinality
 
-// Add adds the other flags to these flags.
-func (c *CostFlags) Add(other CostFlags) {
-	c.FullScanPenalty = c.FullScanPenalty || other.FullScanPenalty
-	c.HugeCostPenalty = c.HugeCostPenalty || other.HugeCostPenalty
-	c.UnboundedCardinality = c.UnboundedCardinality || other.UnboundedCardinality
-}
-
-// Empty returns true if these flags are empty.
-func (c CostFlags) Empty() bool {
-	return !c.FullScanPenalty && !c.HugeCostPenalty && !c.UnboundedCardinality
-}
+	// NoPenalties represents no penalties.
+	NoPenalties CostFlags = 0
+)

pkg/sql/opt/memo/cost_test.go

@@ -26,17 +26,17 @@ func TestCostLess(t *testing.T) {
 		{Cost{C: 1}, Cost{C: 1.00000001}, true},
 		{Cost{C: 1000}, Cost{C: 1000.00000000001}, false},
 		{Cost{C: 1000}, Cost{C: 1000.00001}, true},
-		{Cost{C: 1.0, Flags: CostFlags{FullScanPenalty: true}}, Cost{C: 1.0}, false},
-		{Cost{C: 1.0}, Cost{C: 1.0, Flags: CostFlags{HugeCostPenalty: true}}, true},
-		{Cost{C: 1.0, Flags: CostFlags{FullScanPenalty: true, HugeCostPenalty: true}}, Cost{C: 1.0}, false},
-		{Cost{C: 1.0, Flags: CostFlags{FullScanPenalty: true}}, Cost{C: 1.0, Flags: CostFlags{HugeCostPenalty: true}}, true},
+		{Cost{C: 1.0, Flags: FullScanPenalty}, Cost{C: 1.0}, false},
+		{Cost{C: 1.0}, Cost{C: 1.0, Flags: HugeCostPenalty}, true},
+		{Cost{C: 1.0, Flags: FullScanPenalty | HugeCostPenalty}, Cost{C: 1.0}, false},
+		{Cost{C: 1.0, Flags: FullScanPenalty}, Cost{C: 1.0, Flags: HugeCostPenalty}, true},
 		{MaxCost, Cost{C: 1.0}, false},
 		{Cost{C: 0.0}, MaxCost, true},
 		{MaxCost, MaxCost, false},
-		{MaxCost, Cost{C: 1.0, Flags: CostFlags{FullScanPenalty: true}}, false},
-		{Cost{C: 1.0, Flags: CostFlags{HugeCostPenalty: true}}, MaxCost, true},
-		{Cost{C: 2.0, Flags: CostFlags{}}, Cost{C: 1.0, Flags: CostFlags{UnboundedCardinality: true}}, true},
-		{Cost{C: 1.0, Flags: CostFlags{UnboundedCardinality: true}}, Cost{C: 2.0, Flags: CostFlags{}}, false},
+		{MaxCost, Cost{C: 1.0, Flags: FullScanPenalty}, false},
+		{Cost{C: 1.0, Flags: HugeCostPenalty}, MaxCost, true},
+		{Cost{C: 2.0}, Cost{C: 1.0, Flags: UnboundedCardinality}, true},
+		{Cost{C: 1.0, Flags: UnboundedCardinality}, Cost{C: 2.0}, false},
 		// Auxiliary information should not affect the comparison.
 		{Cost{C: 1.0, aux: testAux{0}}, Cost{C: 1.0, aux: testAux{1}}, false},
 	}
@@ -55,8 +55,9 @@ func TestCostAdd(t *testing.T) {
 		{Cost{C: 0.0}, Cost{C: 0.0}, Cost{C: 0.0}},
 		{Cost{C: -1.0}, Cost{C: 1.0}, Cost{C: 0.0}},
 		{Cost{C: 1.5}, Cost{C: 2.5}, Cost{C: 4.0}},
-		{Cost{C: 1.0, Flags: CostFlags{FullScanPenalty: true}}, Cost{C: 2.0}, Cost{C: 3.0, Flags: CostFlags{FullScanPenalty: true}}},
-		{Cost{C: 1.0}, Cost{C: 2.0, Flags: CostFlags{HugeCostPenalty: true}}, Cost{C: 3.0, Flags: CostFlags{HugeCostPenalty: true}}},
+		{Cost{C: 1.0, Flags: FullScanPenalty}, Cost{C: 2.0}, Cost{C: 3.0, Flags: FullScanPenalty}},
+		{Cost{C: 1.0}, Cost{C: 2.0, Flags: HugeCostPenalty}, Cost{C: 3.0, Flags: HugeCostPenalty}},
+		{Cost{C: 1.0, Flags: UnboundedCardinality}, Cost{C: 2.0, Flags: HugeCostPenalty}, Cost{C: 3.0, Flags: HugeCostPenalty | UnboundedCardinality}},
 		{Cost{C: 1.0, aux: testAux{1}}, Cost{C: 1.0, aux: testAux{2}}, Cost{C: 2.0, aux: testAux{3}}},
 		{Cost{C: 1.0, aux: testAux{200}}, Cost{C: 1.0, aux: testAux{100}}, Cost{C: 2.0, aux: testAux{255}}},
 	}
@@ -67,41 +68,3 @@ func TestCostAdd(t *testing.T) {
 		}
 	}
 }
-
-func TestCostFlagsLess(t *testing.T) {
-	testCases := []struct {
-		left, right CostFlags
-		expected    bool
-	}{
-		{CostFlags{FullScanPenalty: false, HugeCostPenalty: false}, CostFlags{FullScanPenalty: true, HugeCostPenalty: true}, true},
-		{CostFlags{FullScanPenalty: true, HugeCostPenalty: true}, CostFlags{FullScanPenalty: false, HugeCostPenalty: false}, false},
-		{CostFlags{FullScanPenalty: true, HugeCostPenalty: true}, CostFlags{FullScanPenalty: true, HugeCostPenalty: true}, false},
-		{CostFlags{FullScanPenalty: false}, CostFlags{FullScanPenalty: true}, true},
-		{CostFlags{HugeCostPenalty: false}, CostFlags{HugeCostPenalty: true}, true},
-		{CostFlags{UnboundedCardinality: false}, CostFlags{UnboundedCardinality: true}, true},
-		{CostFlags{UnboundedCardinality: true}, CostFlags{UnboundedCardinality: false}, false},
-	}
-	for _, tc := range testCases {
-		if tc.left.Less(tc.right) != tc.expected {
-			t.Errorf("expected %v.Less(%v) to be %v", tc.left, tc.right, tc.expected)
-		}
-	}
-}
-
-func TestCostFlagsAdd(t *testing.T) {
-	testCases := []struct {
-		left, right, expected CostFlags
-	}{
-		{CostFlags{FullScanPenalty: false, HugeCostPenalty: false}, CostFlags{FullScanPenalty: true, HugeCostPenalty: true}, CostFlags{FullScanPenalty: true, HugeCostPenalty: true}},
-		{CostFlags{FullScanPenalty: true, HugeCostPenalty: true}, CostFlags{FullScanPenalty: false, HugeCostPenalty: false}, CostFlags{FullScanPenalty: true, HugeCostPenalty: true}},
-		{CostFlags{FullScanPenalty: false}, CostFlags{FullScanPenalty: true}, CostFlags{FullScanPenalty: true}},
-		{CostFlags{HugeCostPenalty: false}, CostFlags{HugeCostPenalty: true}, CostFlags{HugeCostPenalty: true}},
-		{CostFlags{FullScanPenalty: true, HugeCostPenalty: false}, CostFlags{FullScanPenalty: false, HugeCostPenalty: true}, CostFlags{FullScanPenalty: true, HugeCostPenalty: true}},
-	}
-	for _, tc := range testCases {
-		tc.left.Add(tc.right)
-		if tc.left != tc.expected {
-			t.Errorf("expected %v.Add(%v) to be %v, got %v", tc.left, tc.right, tc.expected, tc.left)
-		}
-	}
-}

pkg/sql/opt/memo/expr_format.go

@@ -919,16 +919,16 @@ func (f *ExprFmtCtx) formatRelational(e RelExpr, tp treeprinter.Node) {
 		if cost.C != 0 {
 			tp.Childf("cost: %.9g", cost.C)
 		}
-		if !cost.Flags.Empty() {
+		if cost.Flags != NoPenalties {
 			var b strings.Builder
 			b.WriteString("cost-flags:")
-			if cost.Flags.FullScanPenalty {
+			if cost.Flags&FullScanPenalty != 0 {
 				b.WriteString(" full-scan-penalty")
 			}
-			if cost.Flags.HugeCostPenalty {
+			if cost.Flags&HugeCostPenalty != 0 {
 				b.WriteString(" huge-cost-penalty")
 			}
-			if cost.Flags.UnboundedCardinality {
+			if cost.Flags&UnboundedCardinality != 0 {
 				b.WriteString(" unbounded-cardinality")
 			}
 			tp.Child(b.String())

pkg/sql/opt/xform/coster.go

@@ -190,7 +190,7 @@ var (
 	// that "violate" a hint like forcing a specific index or join algorithm.
 	// If the final expression has this cost or larger, it means that there was no
 	// plan that could satisfy the hints.
-	hugeCost = memo.Cost{C: 1e100, Flags: memo.CostFlags{HugeCostPenalty: true}}
+	hugeCost = memo.Cost{C: 1e100, Flags: memo.HugeCostPenalty}
 
 	// SmallDistributeCost is the per-operation cost overhead for scans which may
 	// access remote regions, but the scanned table is unpartitioned with no lease
@@ -218,7 +218,7 @@ var (
 	//               the correct plan is found?
 	LargeDistributeCostWithHomeRegion = memo.Cost{
 		C:     LargeDistributeCost.C / 2,
-		Flags: memo.CostFlags{HugeCostPenalty: true},
+		Flags: memo.HugeCostPenalty,
 	}
 )
 
@@ -645,7 +645,7 @@ func (c *coster) ComputeCost(candidate memo.RelExpr, required *physical.Required
 	if candidate.Relational().Cardinality.IsUnbounded() {
 		cost.C += cpuCostFactor
 		if c.evalCtx.SessionData().OptimizerPreferBoundedCardinality {
-			cost.Flags.UnboundedCardinality = true
+			cost.Flags |= memo.UnboundedCardinality
 		}
 	}
 
@@ -909,7 +909,7 @@ func (c *coster) computeScanCost(scan *memo.ScanExpr, required *physical.Require
 		cost.IncrFullScanCount()
 		if scan.Flags.AvoidFullScan {
 			// Apply a penalty for a full scan if needed.
-			cost.Flags.FullScanPenalty = true
+			cost.Flags |= memo.FullScanPenalty
 		}
 	}
 

pkg/sql/prep/statement.go

@@ -187,10 +187,9 @@ func (p *planCosts) NumCustom() int {
 // average cost of the custom plans.
 func (p *planCosts) IsGenericOptimal() bool {
 	// Check cost flags and full scan counts.
-	if gc := p.generic.FullScanCount(); gc > 0 || !p.generic.Flags.Empty() {
+	if gc := p.generic.FullScanCount(); gc > 0 || p.generic.Flags != memo.NoPenalties {
 		for i := 0; i < p.custom.length; i++ {
-			if p.custom.costs[i].Flags.Less(p.generic.Flags) ||
-				gc > p.custom.costs[i].FullScanCount() {
+			if p.custom.costs[i].Flags < p.generic.Flags || gc > p.custom.costs[i].FullScanCount() {
 				return false
 			}
 		}