[CALCITE-5347] Add 'SELECT ... BY', a syntax extension that is shorthand for GROUP BY and ORDER BY

Author: xiedeyantu

babel/src/main/codegen/config.fmpp

@@ -119,7 +119,7 @@ data: {
       "BOOLEAN"
       "BOTH"
       "BREADTH"
-      "BY"
+#     "BY"
 #     "CALL"
       "CALLED"
       "CARDINALITY"
@@ -618,6 +618,7 @@ data: {
     includeParsingStringLiteralAsArrayLiteral: true
     includeIntervalWithoutQualifier: true
     includeStarExclude: true
+    includeSelectBy: true
   }
 }
 

babel/src/test/java/org/apache/calcite/test/BabelTest.java

@@ -17,15 +17,27 @@
 package org.apache.calcite.test;
 
 import org.apache.calcite.config.CalciteConnectionProperty;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.rel2sql.RelToSqlConverter;
 import org.apache.calcite.rel.type.DelegatingTypeSystem;
 import org.apache.calcite.rel.type.RelDataTypeField;
 import org.apache.calcite.rel.type.TimeFrameSet;
+import org.apache.calcite.schema.SchemaPlus;
+import org.apache.calcite.sql.SqlDialect;
+import org.apache.calcite.sql.SqlNode;
 import org.apache.calcite.sql.SqlOperatorTable;
+import org.apache.calcite.sql.dialect.CalciteSqlDialect;
 import org.apache.calcite.sql.fun.SqlLibrary;
 import org.apache.calcite.sql.fun.SqlLibraryOperatorTableFactory;
+import org.apache.calcite.sql.parser.SqlParser;
 import org.apache.calcite.sql.parser.SqlParserFixture;
 import org.apache.calcite.sql.parser.babel.SqlBabelParserImpl;
 import org.apache.calcite.sql.validate.SqlConformanceEnum;
+import org.apache.calcite.tools.FrameworkConfig;
+import org.apache.calcite.tools.Frameworks;
+import org.apache.calcite.tools.Planner;
+import org.apache.calcite.tools.Programs;
+import org.apache.calcite.util.TestUtil;
 
 import com.google.common.collect.ImmutableList;
 
@@ -43,6 +55,8 @@
 import java.util.function.UnaryOperator;
 import java.util.stream.Collectors;
 
+import static org.apache.calcite.test.Matchers.isLinux;
+
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.MatcherAssert.assertThat;
 
@@ -341,6 +355,133 @@ names, is(
         .type("RecordType(VARCHAR(10) NOT NULL NAME) NOT NULL");
   }
 
+  /** Test case for
+   * <a href="https://issues.apache.org/jira/browse/CALCITE-5347">[CALCITE-5347]
+   * Support parse "SELECT ... BY" in Babel parser</a>. */
+  @Test void testByClause() {
+    final SqlValidatorFixture v = Fixtures.forValidator()
+        .withParserConfig(c -> c.withParserFactory(SqlBabelParserImpl.FACTORY))
+        .withConformance(SqlConformanceEnum.BABEL);
+
+    // Test basic BY clause: SELECT a BY b is sugar for SELECT b, a GROUP BY b ORDER BY b
+    v.withSql("select ename, empno by deptno from emp").ok();
+    // Test BY clause with alias
+    v.withSql("select ename, empno by deptno as dept from emp").ok();
+    // Test BY clause with DESC modifier
+    v.withSql("select ename, empno by deptno DESC from emp").ok();
+    // Test BY clause with multiple columns
+    v.withSql("select ename, empno by deptno, job from emp").ok();
+    // Test complex BY clause example from the feature proposal
+    v.withSql("select e.ename, e.empno by d.name as dept DESC, e.job as title "
+        + "from emp as e join dept as d on e.deptno = d.deptno where d.name = 'SALES'")
+        .ok();
+
+    // Test SELECT BY cannot be used with GROUP BY
+    v.withSql("select ename by deptno from emp ^group by empno^")
+        .fails("SELECT BY cannot be used with GROUP BY");
+    // Test SELECT BY cannot be used with ORDER BY
+    v.withSql("select ename by deptno from emp ^order by empno^")
+        .fails("SELECT BY cannot be used with ORDER BY");
+  }
+
+  /** Test case of
+   * <a href="https://issues.apache.org/jira/browse/CALCITE-5347">[CALCITE-5347]
+   * Add 'SELECT ... BY', a syntax extension that is shorthand for GROUP BY and ORDER BY</a>. */
+  @Test void testByClauseConversion() {
+    // Test basic BY clause: SELECT a BY b is sugar for SELECT b, a GROUP BY b ORDER BY b
+    final String sql = "select ename, empno by deptno from emp";
+    final String expected = "SELECT \"DEPTNO\","
+        + " ANY_VALUE(\"ENAME\") AS \"ENAME\","
+        + " ANY_VALUE(\"EMPNO\") AS \"EMPNO\"\n"
+        + "FROM \"SCOTT\".\"EMP\"\n"
+        + "GROUP BY \"DEPTNO\"\n"
+        + "ORDER BY \"DEPTNO\"";
+    checkSqlConversion(sql, expected);
+
+    // Test BY clause with alias
+    final String sql2 = "select ename, empno by deptno as dept from emp";
+    final String expected2 = "SELECT \"DEPTNO\" AS \"DEPT\","
+        + " ANY_VALUE(\"ENAME\") AS \"ENAME\","
+        + " ANY_VALUE(\"EMPNO\") AS \"EMPNO\"\n"
+        + "FROM \"SCOTT\".\"EMP\"\n"
+        + "GROUP BY \"DEPTNO\"\n"
+        + "ORDER BY \"DEPTNO\"";
+    checkSqlConversion(sql2, expected2);
+
+    // Test BY clause with DESC modifier
+    final String sql3 = "select ename, empno by deptno DESC from emp";
+    final String expected3 = "SELECT \"DEPTNO\","
+        + " ANY_VALUE(\"ENAME\") AS \"ENAME\","
+        + " ANY_VALUE(\"EMPNO\") AS \"EMPNO\"\n"
+        + "FROM \"SCOTT\".\"EMP\"\n"
+        + "GROUP BY \"DEPTNO\"\n"
+        + "ORDER BY \"DEPTNO\" DESC";
+    checkSqlConversion(sql3, expected3);
+
+    // Test BY clause with multiple columns
+    final String sql4 = "select ename, empno by deptno, job from emp";
+    final String expected4 = "SELECT \"DEPTNO\", \"JOB\","
+        + " ANY_VALUE(\"ENAME\") AS \"ENAME\","
+        + " ANY_VALUE(\"EMPNO\") AS \"EMPNO\"\n"
+        + "FROM \"SCOTT\".\"EMP\"\n"
+        + "GROUP BY \"DEPTNO\", \"JOB\"\n"
+        + "ORDER BY \"DEPTNO\", \"JOB\"";
+    checkSqlConversion(sql4, expected4);
+
+    // Test complex BY clause example from the feature proposal
+    final String sql5 = "SELECT e.ename, e.empno BY d.dname AS dept DESC, e.job AS title\n"
+        + "FROM emp AS e\n"
+        + "  JOIN dept AS d ON e.deptno = d.deptno\n"
+        + "WHERE d.loc = 'CHICAGO'";
+    final String expected5 = "SELECT \"DEPT\".\"DNAME\" AS \"DEPT\","
+        + " \"EMP\".\"JOB\" AS \"TITLE\","
+        + " ANY_VALUE(\"EMP\".\"ENAME\") AS \"ENAME\","
+        + " ANY_VALUE(\"EMP\".\"EMPNO\") AS \"EMPNO\"\n"
+        + "FROM \"SCOTT\".\"EMP\"\n"
+        + "INNER JOIN \"SCOTT\".\"DEPT\" ON \"EMP\".\"DEPTNO\" = \"DEPT\".\"DEPTNO\"\n"
+        + "WHERE \"DEPT\".\"LOC\" = 'CHICAGO'\n"
+        + "GROUP BY \"DEPT\".\"DNAME\", \"EMP\".\"JOB\"\n"
+        + "ORDER BY \"DEPT\".\"DNAME\" DESC, \"EMP\".\"JOB\"";
+    checkSqlConversion(sql5, expected5);
+  }
+
+  private void checkSqlConversion(String sql, String expected) {
+    try {
+      final SqlParser.Config parserConfig = SqlParser.config()
+          .withParserFactory(SqlBabelParserImpl.FACTORY);
+
+      final SchemaPlus rootSchema = Frameworks.createRootSchema(true);
+      final SchemaPlus defaultSchema =
+          CalciteAssert.addSchema(rootSchema, CalciteAssert.SchemaSpec.JDBC_SCOTT);
+
+      final FrameworkConfig config = Frameworks.newConfigBuilder()
+          .parserConfig(parserConfig)
+          .defaultSchema(defaultSchema)
+          .programs(Programs.standard())
+          .build();
+
+      final Planner planner = Frameworks.getPlanner(config);
+      final SqlNode parse = planner.parse(sql);
+      final SqlNode validate = planner.validate(parse);
+      RelNode rel = planner.rel(validate).project();
+
+      final SqlDialect dialect = CalciteSqlDialect.DEFAULT;
+      final RelToSqlConverter converter = new RelToSqlConverter(dialect);
+      final SqlNode sqlNode = converter.visitRoot(rel).asStatement();
+      final String actual = sqlNode.toSqlString(c ->
+          c.withDialect(dialect)
+              .withAlwaysUseParentheses(false)
+              .withSelectListItemsOnSeparateLines(false)
+              .withUpdateSetListNewline(false)
+              .withIndentation(0))
+          .getSql();
+
+      assertThat(actual, isLinux(expected));
+    } catch (Exception e) {
+      throw TestUtil.rethrow(e);
+    }
+  }
+
   private void checkSqlResult(String funLibrary, String query, String result) {
     CalciteAssert.that()
         .with(CalciteConnectionProperty.PARSER_FACTORY,

babel/src/test/resources/sql/select.iq

@@ -160,4 +160,78 @@ from emp e join dept d on e.deptno = d.deptno limit 1;
 
 !ok
 
+# Test basic BY clause: SELECT a BY b is sugar for SELECT b, a GROUP BY b ORDER BY b
+select ename, empno by deptno from emp;
++--------+--------+-------+
+| DEPTNO | ENAME  | EMPNO |
++--------+--------+-------+
+|     10 | MILLER |  7934 |
+|     20 | SMITH  |  7902 |
+|     30 | WARD   |  7900 |
++--------+--------+-------+
+(3 rows)
+
+!ok
+
+# Test BY clause with alias
+select ename, empno by deptno as dept from emp;
++------+--------+-------+
+| DEPT | ENAME  | EMPNO |
++------+--------+-------+
+|   10 | MILLER |  7934 |
+|   20 | SMITH  |  7902 |
+|   30 | WARD   |  7900 |
++------+--------+-------+
+(3 rows)
+
+!ok
+
+# Test BY clause with DESC modifier
+select ename, empno by deptno DESC from emp;
++--------+--------+-------+
+| DEPTNO | ENAME  | EMPNO |
++--------+--------+-------+
+|     30 | WARD   |  7900 |
+|     20 | SMITH  |  7902 |
+|     10 | MILLER |  7934 |
++--------+--------+-------+
+(3 rows)
+
+!ok
+
+# Test BY clause with multiple columns
+select ename, empno by deptno, job from emp;
++--------+-----------+--------+-------+
+| DEPTNO | JOB       | ENAME  | EMPNO |
++--------+-----------+--------+-------+
+|     10 | CLERK     | MILLER |  7934 |
+|     10 | MANAGER   | CLARK  |  7782 |
+|     10 | PRESIDENT | KING   |  7839 |
+|     20 | ANALYST   | SCOTT  |  7902 |
+|     20 | CLERK     | SMITH  |  7876 |
+|     20 | MANAGER   | JONES  |  7566 |
+|     30 | CLERK     | JAMES  |  7900 |
+|     30 | MANAGER   | BLAKE  |  7698 |
+|     30 | SALESMAN  | WARD   |  7844 |
++--------+-----------+--------+-------+
+(9 rows)
+
+!ok
+
+# Test complex BY clause example from the feature proposal
+SELECT e.ename, e.empno BY d.dname AS dept DESC, e.job AS title
+FROM emp AS e
+  JOIN dept AS d ON e.deptno = d.deptno
+WHERE d.loc = 'CHICAGO';
++-------+----------+-------+-------+
+| DEPT  | TITLE    | ENAME | EMPNO |
++-------+----------+-------+-------+
+| SALES | CLERK    | JAMES |  7900 |
+| SALES | MANAGER  | BLAKE |  7698 |
+| SALES | SALESMAN | WARD  |  7844 |
++-------+----------+-------+-------+
+(3 rows)
+
+!ok
+
 # End select.iq

core/src/main/codegen/templates/Parser.jj

@@ -90,6 +90,7 @@ import org.apache.calcite.sql.SqlPrefixOperator;
 import org.apache.calcite.sql.SqlRowTypeNameSpec;
 import org.apache.calcite.sql.SqlSampleSpec;
 import org.apache.calcite.sql.SqlSelect;
+import org.apache.calcite.sql.SqlByRewriter;
 import org.apache.calcite.sql.SqlSelectKeyword;
 import org.apache.calcite.sql.SqlStarExclude;
 import org.apache.calcite.sql.SqlSetOption;
@@ -724,6 +725,12 @@ SqlNode OrderByLimitOpt(SqlNode e) :
     ]
     {
         if (orderBy != null || offsetFetch[0] != null || offsetFetch[1] != null) {
+            if (orderBy != null
+                && e instanceof SqlSelect
+                && ((SqlSelect) e).hasByClause()) {
+              throw SqlUtil.newContextException(orderBy.getParserPosition(),
+                  RESOURCE.selectByCannotWithOrderBy());
+            }
             return new SqlOrderBy(getPos(), e,
                 Util.first(orderBy, SqlNodeList.EMPTY),
                 offsetFetch[0], offsetFetch[1]);
@@ -1343,6 +1350,7 @@ SqlSelect SqlSelect() :
     final SqlNode having;
     final SqlNodeList windowDecls;
     final SqlNode qualify;
+    final SqlNodeList by;
     final List<SqlNode> hints = new ArrayList<SqlNode>();
     final Span s;
 }
@@ -1363,6 +1371,11 @@ SqlSelect SqlSelect() :
     }
     AddSelectItem(selectList)
     ( <COMMA> AddSelectItem(selectList) )*
+<#if parser.includeSelectBy!false>
+    ( by = SqlSelectBy() | { by = null; } )
+<#else>
+    { by = null; }
+</#if>
     (
         <FROM> fromClause = FromClause()
         ( where = Where() | { where = null; } )
@@ -1381,10 +1394,12 @@ SqlSelect SqlSelect() :
         }
     )
     {
-        return new SqlSelect(s.end(this), keywordList,
+        final SqlSelect select = new SqlSelect(s.end(this), keywordList,
             new SqlNodeList(selectList, Span.of(selectList).pos()),
             fromClause, where, groupBy, having, windowDecls, qualify,
             null, null, null, new SqlNodeList(hints, getPos()));
+        SqlByRewriter.rewrite(select, by);
+        return select;
     }
 }
 
@@ -2969,26 +2984,65 @@ SqlNodeList OrderBy(boolean accept) :
             throw SqlUtil.newContextException(s.pos(), RESOURCE.illegalOrderBy());
         }
     }
-    <BY> AddOrderItem(list)
-    (
-        // NOTE jvs 6-Feb-2004:  See comments at top of file for why
-        // hint is necessary here.
-        LOOKAHEAD(2) <COMMA> AddOrderItem(list)
-    )*
+    <BY> OrderItemList(list)
+    {
+        return new SqlNodeList(list, s.addAll(list).pos());
+    }
+}
+
+<#if parser.includeSelectBy!false>
+/**
+ * Parses a BY clause for SELECT (syntactic sugar for GROUP BY ... ORDER BY).
+ */
+SqlNodeList SqlSelectBy() :
+{
+    final List<SqlNode> list = new ArrayList<SqlNode>();
+    final Span s;
+}
+{
+    <BY> { s = span(); }
+    OrderItemList(list)
     {
         return new SqlNodeList(list, s.addAll(list).pos());
     }
 }
+</#if>
+
+
+/**
+ * Parses a list of ORDER BY items.
+ */
+void OrderItemList(List<SqlNode> list) :
+{
+}
+{
+    AddOrderItem(list)
+    (
+        LOOKAHEAD(2) <COMMA> AddOrderItem(list)
+    )*
+}
 
 /**
  * Parses one item in an ORDER BY clause, and adds it to a list.
  */
 void AddOrderItem(List<SqlNode> list) :
 {
     SqlNode e;
+    final SqlIdentifier id;
 }
 {
     e = Expression(ExprContext.ACCEPT_SUB_QUERY)
+    (
+        [ <AS>
+            (
+                id = SimpleIdentifier()
+            |
+                LOOKAHEAD(1)
+                id = SimpleIdentifierFromStringLiteral()
+            )
+            { e = SqlStdOperatorTable.AS.createCall(span().end(e), e, id); }
+        ]
+    )
     (
         <ASC>
     |   <DESC> {