fix group by concat with order by subqueries

Author: James Cor

enginetest/memory_engine_test.go

@@ -202,24 +202,40 @@ func TestSingleQueryPrepared(t *testing.T) {
 
 // Convenience test for debugging a single query. Unskip and set to the desired query.
 func TestSingleScript(t *testing.T) {
-	t.Skip()
+	//t.Skip()
 	var scripts = []queries.ScriptTest{
 		{
-			Name:        "AS OF propagates to nested CALLs",
-			SetUpScript: []string{},
+			Name:    "Group Concat with Subquery in ORDER BY",
+			Dialect: "mysql",
+			SetUpScript: []string{
+				"CREATE TABLE test_data (id INT PRIMARY KEY, name VARCHAR(50), age INT, category VARCHAR(10))",
+				`INSERT INTO test_data VALUES  
+(1, 'Alice', 25, 'A'),
+(2, 'Bob', 30, 'B'), 
+(3, 'Charlie', 22, 'A'), 
+(4, 'Diana', 28, 'C'), 
+(5, 'Eve', 35, 'B'), 
+(6, 'Frank', 26, 'A')`,
+			},
 			Assertions: []queries.ScriptTestAssertion{
+				//{
+				//	Query:    "SELECT group_concat(name order by age) from test_data",
+				//	Expected: []sql.Row{},
+				//},
 				{
-					Query: "create procedure create_proc() create table t (i int primary key, j int);",
-					Expected: []sql.Row{
-						{types.NewOkResult(0)},
-					},
-				},
-				{
-					Query: "call create_proc()",
-					Expected: []sql.Row{
-						{types.NewOkResult(0)},
-					},
+					//Skip:     true,
+					Query:    "SELECT group_concat(name ORDER BY (SELECT sum(t2.age) FROM test_data t2 WHERE t2.age < test_data.age)) FROM test_data",
+					Expected: []sql.Row{},
 				},
+				//{
+				//	Skip:     true,
+				//	Query:    "SELECT group_concat(name ORDER BY (SELECT AVG(age) FROM test_data t2 WHERE t2.category = test_data.category), id) FROM test_data;",
+				//	Expected: []sql.Row{{"Alice,Charlie,Frank,Diana,Bob,Eve"}},
+				//},
+				//{
+				//	Query:    "SELECT category, group_concat(name ORDER BY (SELECT MAX(age) FROM test_data t2 WHERE t2.id <= test_data.id)) FROM test_data GROUP BY category ORDER BY category",
+				//	Expected: []sql.Row{{"A", "Alice,Charlie,Frank"}, {"B", "Bob,Eve"}, {"C", "Diana"}},
+				//},
 			},
 		},
 	}
@@ -232,8 +248,8 @@ func TestSingleScript(t *testing.T) {
 			panic(err)
 		}
 
-		//engine.EngineAnalyzer().Debug = true
-		//engine.EngineAnalyzer().Verbose = true
+		engine.EngineAnalyzer().Debug = true
+		engine.EngineAnalyzer().Verbose = true
 
 		enginetest.TestScriptWithEngine(t, engine, harness, test)
 	}

enginetest/queries/script_queries.go

@@ -2775,12 +2775,12 @@ CREATE TABLE tab3 (
 		},
 		Assertions: []ScriptTestAssertion{
 			{
-				Skip:     true,
+				//Skip:     true,
 				Query:    "SELECT category, group_concat(name ORDER BY (SELECT COUNT(*) FROM test_data t2 WHERE t2.category = test_data.category AND t2.age < test_data.age)) FROM test_data GROUP BY category ORDER BY category",
 				Expected: []sql.Row{{"A", "Charlie,Alice,Frank"}, {"B", "Bob,Eve"}, {"C", "Diana"}},
 			},
 			{
-				Skip:     true,
+				//Skip:     true,
 				Query:    "SELECT group_concat(name ORDER BY (SELECT AVG(age) FROM test_data t2 WHERE t2.category = test_data.category), id) FROM test_data;",
 				Expected: []sql.Row{{"Alice,Charlie,Frank,Diana,Bob,Eve"}},
 			},
@@ -2804,22 +2804,22 @@ CREATE TABLE tab3 (
 		},
 		Assertions: []ScriptTestAssertion{
 			{
-				Skip:     true,
+				//Skip:     true,
 				Query:    "SELECT category_id, GROUP_CONCAT(name ORDER BY (SELECT rating FROM suppliers WHERE suppliers.id = products.supplier_id) DESC, id ASC) FROM products GROUP BY category_id ORDER BY category_id",
 				Expected: []sql.Row{{1, "Laptop,Keyboard,Mouse,Monitor"}, {2, "Chair,Desk"}},
 			},
 			{
-				Skip:     true,
+				//Skip:     true,
 				Query:    "SELECT GROUP_CONCAT(name ORDER BY (SELECT COUNT(*) FROM products p2 WHERE p2.price < products.price), id) FROM products",
 				Expected: []sql.Row{{"Mouse,Keyboard,Chair,Monitor,Desk,Laptop"}},
 			},
 			{
-				Skip:     true,
+				//Skip:     true,
 				Query:    "SELECT category_id, GROUP_CONCAT(DISTINCT supplier_id ORDER BY (SELECT rating FROM suppliers WHERE suppliers.id = products.supplier_id)) FROM products GROUP BY category_id",
 				Expected: []sql.Row{{1, "2,1"}, {2, "3"}},
 			},
 			{
-				Skip:     true,
+				//Skip:     true,
 				Query:    "SELECT GROUP_CONCAT(name ORDER BY (SELECT priority FROM categories WHERE categories.id = products.category_id), price) FROM products",
 				Expected: []sql.Row{{"Mouse,Keyboard,Monitor,Laptop,Chair,Desk"}},
 			},
@@ -2905,13 +2905,13 @@ CREATE TABLE tab3 (
 			},
 			{
 				// Test with subquery using LIMIT
-				Skip:     true,
+				//Skip:     true,
 				Query:    "SELECT GROUP_CONCAT(data ORDER BY (SELECT weight FROM perf_test p2 WHERE p2.id = perf_test.id LIMIT 1)) FROM perf_test",
 				Expected: []sql.Row{{"C,A,E,B,D"}},
 			},
 			{
 				// Test with very small decimal differences in ORDER BY subquery
-				Skip:     true,
+				//Skip:     true,
 				Query:    "SELECT GROUP_CONCAT(data ORDER BY (SELECT weight + 0.001 * perf_test.id FROM perf_test p2 WHERE p2.id = perf_test.id)) FROM perf_test",
 				Expected: []sql.Row{{"C,A,E,B,D"}},
 			},

sql/analyzer/fix_exec_indexes.go

@@ -16,6 +16,7 @@ package analyzer
 
 import (
 	"fmt"
+	"github.com/dolthub/go-mysql-server/sql/expression/function/aggregation"
 	"strings"
 
 	"github.com/dolthub/go-mysql-server/sql"
@@ -578,9 +579,21 @@ func (s *idxScope) visitSelf(n sql.Node) error {
 		}
 		if ne, ok := n.(sql.Expressioner); ok {
 			scope := append(s.parentScopes, s.childScopes...)
+			// default nodes can't see lateral join nodes, unless we're in lateral
+			// join and lateral scopes are promoted to parent status
 			for _, e := range ne.Expressions() {
-				// default nodes can't see lateral join nodes, unless we're in lateral
-				// join and lateral scopes are promoted to parent status
+				// groupConcat is special as it appends results to outer scope row.
+				if gc, isGc := e.(*aggregation.GroupConcat); isGc {
+					selExprs := gc.GetSelectExprs()
+					selScope := &idxScope{}
+					for _, expr := range selExprs {
+						selScope.columns = append(selScope.columns, expr.String())
+						if gf, isGf := expr.(*expression.GetField); isGf {
+							selScope.ids = append(selScope.ids, gf.Id())
+						}
+					}
+					scope = append(scope, selScope)
+				}
 				s.expressions = append(s.expressions, fixExprToScope(e, scope...))
 			}
 		}
@@ -734,6 +747,10 @@ func fixExprToScope(e sql.Expression, scopes ...*idxScope) sql.Expression {
 			//  this error for the case of DEFAULT in a `plan.Values`, since we analyze the insert source in isolation (we
 			//  don't have the destination schema, and column references in default values are determined in the build phase)
 
+			if e.Name() == "sum(t2.age)" {
+				print()
+			}
+
 			idx, _ := newScope.getIdxId(e.Id(), e.String())
 			if idx >= 0 {
 				return e.WithIndex(idx), transform.NewTree, nil

sql/expression/function/aggregation/group_concat.go

@@ -153,6 +153,40 @@ func (g *GroupConcat) String() string {
 	return sb.String()
 }
 
+func (g *GroupConcat) DebugString() string {
+	sb := strings.Builder{}
+	sb.WriteString("group_concat(")
+	if g.distinct != "" {
+		sb.WriteString(fmt.Sprintf("distinct %s", g.distinct))
+	}
+
+	if g.selectExprs != nil {
+		var exprs = make([]string, len(g.selectExprs))
+		for i, expr := range g.selectExprs {
+			exprs[i] = sql.DebugString(expr)
+		}
+
+		sb.WriteString(strings.Join(exprs, ", "))
+	}
+
+	if len(g.sf) > 0 {
+		sb.WriteString(" order by ")
+		for i, ob := range g.sf {
+			if i > 0 {
+				sb.WriteString(", ")
+			}
+			sb.WriteString(sql.DebugString(ob))
+		}
+	}
+
+	sb.WriteString(" separator ")
+	sb.WriteString(fmt.Sprintf("'%s'", g.separator))
+
+	sb.WriteString(")")
+
+	return sb.String()
+}
+
 // Type implements the Expression interface.
 // cc: https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_group-concat for explanations
 // on return type.
@@ -195,6 +229,12 @@ func (g *GroupConcat) WithChildren(children ...sql.Expression) (sql.Expression,
 	return NewGroupConcat(g.distinct, g.sf.FromExpressions(orderByExpr...), g.separator, children[sortFieldMarker:], g.maxLen), nil
 }
 
+// GetSelectExprs returns the select expressions
+// TODO: just expose the member variable
+func (g *GroupConcat) GetSelectExprs() []sql.Expression {
+	return g.selectExprs
+}
+
 type groupConcatBuffer struct {
 	gc          *GroupConcat
 	rows        []sql.Row
@@ -257,7 +297,8 @@ func (g *groupConcatBuffer) Update(ctx *sql.Context, originalRow sql.Row) error
 
 	// Append the current value to the end of the row. We want to preserve the row's original structure for
 	// for sort ordering in the final step.
-	g.rows = append(g.rows, append(originalRow, nil, vs))
+	// TODO: why nil spacer?
+	g.rows = append(g.rows, append(originalRow, vs))
 
 	return nil
 }