Add skipping clause to query language

Author: [email protected]

src/main/java/com/google/visualization/datasource/QuerySplitter.java

@@ -162,6 +162,7 @@ private static QueryPair splitSQL(Query query) {
       dataSourceQuery.setUserFormatOptions(null);
 
       try {
+        dataSourceQuery.setRowSkipping(0);
         dataSourceQuery.setRowLimit(-1);
         dataSourceQuery.setRowOffset(0);
       } catch (InvalidQueryException e) {
@@ -224,11 +225,21 @@ private static QueryPair splitSQL(Query query) {
 
       completionQuery.setSelection(completionSelection);
     } else {
-      // When there is no pivoting, sql does everything (except options, labels, format).
+      // When there is no pivoting, sql does everything (except skipping, options, labels, format).
       dataSourceQuery.copyFrom(query);
       dataSourceQuery.setOptions(null);
       completionQuery.setOptions(query.getOptions());
       try {
+        // If there is skipping pagination should be done in the completion query
+        if (query.hasRowSkipping()) {
+          dataSourceQuery.setRowSkipping(0);
+          dataSourceQuery.setRowLimit(-1);
+          dataSourceQuery.setRowOffset(0);
+          
+          completionQuery.copyRowSkipping(query);
+          completionQuery.copyRowLimit(query);
+          completionQuery.copyRowOffset(query); 
+        }
         if (query.hasLabels()) {
           dataSourceQuery.setLabels(null);
           QueryLabels labels = query.getLabels();
@@ -256,9 +267,9 @@ private static QueryPair splitSQL(Query query) {
 
   /**
    * Splits the query for a data source with capabilities SORT_AND_PAGINATION.
-   * Algorithm: if the query has filter, grouping, or pivoting requirements the query is
+   * Algorithm: if the query has filter, grouping, pivoting or skipping requirements the query is
    * split as in the NONE case.
-   * If the query does not have filter, grouping, or pivoting the data source query
+   * If the query does not have filter, grouping, pivoting or skipping the data source query
    * receives any sorting or pagination requirements and the completion query receives
    * any selection requirements.
    *
@@ -279,9 +290,21 @@ private static QueryPair splitSortAndPagination(Query query) {
       // The query is copied to the completion query.
       completionQuery.copyFrom(query);
     } else {
-      dataSourceQuery.setSort(query.getSort());
-      dataSourceQuery.copyRowLimit(query);
-      dataSourceQuery.copyRowOffset(query);
+      // The execution order of the 3 relevant operators is:
+      // sort -> skip -> paginate (limit and offset).
+      // Skipping is not a possible data source capability, Therefore:
+      // 1. Sorting can be performed in the data source query.
+      // 2. Pagination should be performed in the data source query IFF skipping
+      //    isn't stated in the original query.
+      dataSourceQuery.setSort(query.getSort());     
+      if (query.hasRowSkipping()) {
+        completionQuery.copyRowSkipping(query);
+        completionQuery.copyRowLimit(query);
+        completionQuery.copyRowOffset(query);
+      } else {
+        dataSourceQuery.copyRowLimit(query);
+        dataSourceQuery.copyRowOffset(query);
+      }
 
       completionQuery.setSelection(query.getSelection());
       completionQuery.setOptions(query.getOptions());

src/main/java/com/google/visualization/datasource/base/ErrorMessages.java

@@ -75,6 +75,7 @@ public class ErrorMessages extends ListResourceBundle {
         "because SELECT contains aggregated columns."},
     {"NO_COL_IN_GROUP_AND_PIVOT", "Column [{0}] cannot appear both in GROUP BY and in PIVOT."},
     {"INVALID_OFFSET", "Invalid value for row offset: {0}"},
+    {"INVALID_SKIPPING", "Invalid value for row skipping: {0}"},
     {"COLUMN_ONLY_ONCE", "Column [{0}] cannot appear more than once in {1}."}
 
 

src/main/java/com/google/visualization/datasource/base/MessagesEnum.java

@@ -147,7 +147,13 @@ public enum MessagesEnum {
    * @param value
    */
    INVALID_OFFSET,
-   
+
+   /**
+    * Invalid value for row skipping.
+    * @param value
+    */
+    INVALID_SKIPPING,
+
    /**
     * Column cannot appear more than once.
     * @param column id
@@ -202,6 +208,8 @@ public enum MessagesEnum {
         "NO_COL_IN_GROUP_AND_PIVOT");
     QUERY_ERROR_TO_MESSAGE.put(MessagesEnum.INVALID_OFFSET,
         "INVALID_OFFSET");
+    QUERY_ERROR_TO_MESSAGE.put(MessagesEnum.INVALID_SKIPPING,
+        "INVALID_SKIPPING");
     QUERY_ERROR_TO_MESSAGE.put(MessagesEnum.COLUMN_ONLY_ONCE,
         "COLUMN_ONLY_ONCE");
   }

src/main/java/com/google/visualization/datasource/query/Query.java

@@ -32,7 +32,7 @@
  * Holds the query data and clauses. This class is the result of parsing the query
  * string that comes from the user.
  * The clauses that can be included in a query are: select, filter, sort, group by, pivot, options,
- * labels, format, limit, and offset.
+ * labels, format, limit, offset and skipping.
  * For more details see the
  * <a href="http://code.google.com/apis/visualization/documentation/querylanguage.html">
  * query language reference
@@ -112,6 +112,17 @@ private static<T> void checkForDuplicates(List<T>
    */
   private QueryPivot pivot = null;
 
+  /**
+   * The number of rows to skip when selecting only a subset of the rows using
+   * skipping clause, e.g., skipping 4.
+   * If a skipping clause, "skip k",  is added to the query, the resulting table
+   * will be consisted of the first row of every k rows of the table returned by
+   * the earlier part of the query, when k corresponds to rowSkipping value.
+   * For example, if rowSkipping = 10 the returned table will consist of rows 0, 10...
+   * The default value is 0, meaning no skipping should be performed.
+   */
+  private int rowSkipping = 0;
+  
   /**
    * Max number of rows to return to caller.
    * If the caller specified this parameter, and the data table returned from
@@ -293,6 +304,59 @@ public boolean hasPivot() {
     return pivot != null && !pivot.getColumnIds().isEmpty();
   }
 
+  /**
+   * Returns the number of rows to skip when selecting only a subset of
+   * the rows using skipping clause, e.g., skipping 4.
+   * The default value is 0, meaning no skipping should be performed.
+   *
+   * @return The number of rows to skip between each row selection.
+   */
+  public int getRowSkipping() {
+    return rowSkipping;
+  }
+
+  /**
+   * Sets the number of rows to skip when selecting only a subset of
+   * the rows using skipping clause, e.g., skipping 4.
+   * The default value is 0, meaning no skipping should be performed.
+   *
+   * If there is is an attempt to set the skipping value to non positive value,
+   * then an InvalidQueryException is thrown.
+   *
+   * @param rowSkipping The number of rows to skip between each row selection.
+   *        0 value means no skipping.
+   *
+   * @throws InvalidQueryException Thrown if an invalid value is specified.
+   */
+  public void setRowSkipping(int rowSkipping) throws InvalidQueryException {
+    if (rowSkipping < 0) {
+      String messageToLogAndUser = MessagesEnum.INVALID_SKIPPING.getMessageWithArgs(
+          localeForUserMessages, Integer.toString(rowSkipping));
+      log.error(messageToLogAndUser);
+      throw new InvalidQueryException(messageToLogAndUser);
+    }
+    this.rowSkipping = rowSkipping;
+  }
+
+  /**
+   * Sets the number of rows to skip when selecting only a subset of
+   * the row, based on another query.
+   * 
+   * @param originalQuery The query from which the row skipping should be taken.
+   */
+  public void copyRowSkipping(Query originalQuery) {
+    rowSkipping = originalQuery.getRowSkipping();
+  }
+
+  /**
+   * Returns true if this query has a row skipping set. A value of 0 means no
+   * skipping.
+   *
+   * @return True if this query has a row skipping set.
+   */
+  public boolean hasRowSkipping() {
+    return rowSkipping > 0;
+  }
 
   /**
    * Returns the maximum number of rows to return to the caller.
@@ -498,8 +562,8 @@ public boolean hasOptions() {
    */
   public boolean isEmpty() {
     return (!hasSort() && !hasSelection() && !hasFilter() && !hasGroup() && !hasPivot()
-        && !hasRowLimit() && !hasRowOffset() && !hasUserFormatOptions()  && !hasLabels()
-        && !hasOptions());
+            && !hasRowSkipping() && !hasRowLimit() && !hasRowOffset()
+            && !hasUserFormatOptions()  && !hasLabels() && !hasOptions());
   }
 
 
@@ -522,6 +586,7 @@ public void copyFrom(Query query) {
     setFilter(query.getFilter());
     setGroup(query.getGroup());
     setPivot(query.getPivot());
+    copyRowSkipping(query);
     copyRowLimit(query);
     copyRowOffset(query);
     setUserFormatOptions(query.getUserFormatOptions());
@@ -925,6 +990,7 @@ public int hashCode() {
     result = prime * result + ((labels == null) ? 0 : labels.hashCode());
     result = prime * result + ((options == null) ? 0 : options.hashCode());
     result = prime * result + ((pivot == null) ? 0 : pivot.hashCode());
+    result = prime * result + rowSkipping;
     result = prime * result + rowLimit;
     result = prime * result + rowOffset;
     result = prime * result + ((selection == null) ? 0 : selection.hashCode());
@@ -980,6 +1046,9 @@ public boolean equals(Object obj) {
     } else if (!pivot.equals(other.pivot)) {
       return false;
     }
+    if (rowSkipping != other.rowSkipping) {
+      return false;
+    }
     if (rowLimit != other.rowLimit) {
       return false;
     }
@@ -1073,6 +1142,9 @@ public String toQueryString() {
     if (hasSort()) {
       clauses.add("ORDER BY " + sort.toQueryString());
     }
+    if (hasRowSkipping()) {
+      clauses.add("SKIPPING " + rowSkipping);
+    }
     if (hasRowLimit()) {
       clauses.add("LIMIT " + rowLimit);
     }

src/main/java/com/google/visualization/datasource/query/engine/QueryEngine.java

@@ -44,6 +44,7 @@
 
 import com.ibm.icu.util.ULocale;
 
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -55,7 +56,7 @@
 
 /**
  * A collection of static methods that perform the operations involved in executing a query,
- * i.e., selection, sorting, paging (limit and offset), grouping and pivoting, filtering, 
+ * i.e., selection, sorting, paging (limit and offset), grouping and pivoting, filtering, skipping,
  * applying labels, and custom formatting. This also takes care of calculated columns.
  * This also takes care of scalar function columns.
  *
@@ -125,6 +126,7 @@ public static DataTable executeQuery(Query query, DataTable table, ULocale local
       table = performFilter(table, query);
       table = performGroupingAndPivoting(table, query, columnIndices, columnLookups);
       table = performSort(table, query, locale);
+      table = performSkipping(table, query);
       table = performPagination(table, query);
 
       AtomicReference<ColumnIndices> columnIndicesReference =
@@ -140,6 +142,41 @@ public static DataTable executeQuery(Query query, DataTable table, ULocale local
     return table;
   }
 
+  /**
+   * Returns a table consisted of a subset of rows of the input table. 
+   * We select the first out of every k rows in the table according to the 
+   * skipping value in the query.
+   * If there is no need to do anything, returns the original table.
+   * 
+   * @param table The original table.
+   * @param query The query.
+   *
+   * @return The skipped table, or the original if no skipping is needed. 
+   */
+  private static DataTable performSkipping(DataTable table, Query query)
+      throws TypeMismatchException {
+    int rowSkipping = query.getRowSkipping();
+    
+    // Return the original table if no skipping is needed
+    if (rowSkipping <= 1) {
+      return table;
+    }
+
+    // Add the first out of every k rows of the original table to TableRow
+    int numRows = table.getNumberOfRows();
+    List<TableRow> relevantRows = new ArrayList<TableRow>();
+    for (int rowIndex = 0; rowIndex < numRows; rowIndex += rowSkipping) {
+        relevantRows.add(table.getRows().get(rowIndex));
+    }
+    
+    // Create a table out of the TableRow array
+    DataTable newTable = new DataTable();
+    newTable.addColumns(table.getColumnDescriptions());
+    newTable.addRows(relevantRows);
+    
+    return newTable;
+  }
+
   /**
    * Returns a paginated table, based on the row limit and offset parameters.
    * If there is no need to do anything, returns the original table.

src/main/java/com/google/visualization/datasource/query/parser/QueryParser.jj

@@ -107,6 +107,7 @@ TOKEN[IGNORE_CASE]:
   | <KW_PIVOT:        "pivot">
   | <KW_ORDER:        "order">
   | <KW_BY:           "by">
+  | <KW_SKIPPING:     "skipping">
   | <KW_LIMIT:        "limit">
   | <KW_OFFSET:       "offset">
   | <KW_LABEL:        "label">
@@ -208,6 +209,7 @@ Query queryStatement() throws InvalidQueryException :
       [ groupByClause(query) ]
       [ pivotClause(query) ]
       [ orderByClause(query) ]
+      [ skippingClause(query) ]
       [ limitClause(query) ]
       [ offsetClause(query) ]
       [ labelClause(query) ]
@@ -294,6 +296,17 @@ void orderByClause(Query query) throws InvalidQueryException :
   { query.setSort(sort); }
 }
 
+// The skipping clause (e.g., SKIPPING 10)
+void skippingClause(Query query) throws InvalidQueryException :
+{
+  int skipping;
+}
+{
+  <KW_SKIPPING>
+  skipping = integerLiteral()
+  { query.setRowSkipping(skipping); }
+}
+
 // The limit clause (e.g., LIMIT 100)
 void limitClause(Query query) throws InvalidQueryException :
 {