LEXIO-37880 Adds support for group by all (#2)

jdrake · ns-circle-ci · web-flow · commit d31033039a10 · 2022-04-14T10:36:45.000-07:00
* all

* version

* Version bumped to 0.3.0

Co-authored-by: ns-circle-ci &lt;devops-team+circleci@narrativescience.com&gt;
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -2,7 +2,7 @@
 default_stages: [commit]
 
 default_language_version:
-  python: python3
+  python: python3.9
 
 exclude: "examples/.*$"
 
@@ -74,7 +74,7 @@ repos:
         entry: mypy -p pysaql
         language: python
         types: [ file, python ]
-        additional_dependencies: [ mypy==0.910 ]
+        additional_dependencies: [ mypy==0.940 ]
         pass_filenames: false
 
       - id: flake8
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pysaql"
-version = "0.2.0"
+version = "0.3.0"
 description = "Python SAQL query builder"
 authors = ["Jonathan Drake <jon.drake@salesforce.com>"]
 license = "BSD-3-Clause"
diff --git a/pysaql/__init__.py b/pysaql/__init__.py
@@ -1,3 +1,3 @@
 """Python SAQL query builder"""
 
-__version__ = "0.2.0"
+__version__ = "0.3.0"
diff --git a/pysaql/aggregation.py b/pysaql/aggregation.py
@@ -1,8 +1,8 @@
 """Contains aggregation function definitions"""
 
-from typing import Optional, Sequence, Tuple, Union
+from __future__ import annotations
 
-from typing_extensions import Self
+from typing import Optional, Sequence, Tuple, Union
 
 from .enums import Order
 from .function import Function, NullaryFunction
@@ -34,7 +34,7 @@ def over(
         row_range: Tuple[Optional[int], Optional[int]],
         reset_groups: Union[str, Sequence[Scalar]],
         order_by: Sequence[Union[Scalar, Tuple[Scalar, Order]]],
-    ) -> Self:
+    ) -> WindowFunction:
         """Set the windowing function parameters
 
         Args:
diff --git a/pysaql/date_.py b/pysaql/date_.py
@@ -329,6 +329,6 @@ def __str__(self) -> str:
             end = ",".join(str(arg) for arg in self.end_date._args)
             return f"[dateRange([{start}], [{end}])]"
         else:
-            start = self.start_date or ""
-            end = self.end_date or ""
+            start = str(self.start_date) if self.start_date else ""
+            end = str(self.end_date) if self.end_date else ""
             return f"[{start}..{end}]"
diff --git a/pysaql/enums.py b/pysaql/enums.py
diff --git a/pysaql/expression.py b/pysaql/expression.py
diff --git a/pysaql/function.py b/pysaql/function.py
@@ -1,14 +1,14 @@
 """Contains core function definitions"""
 
-from typing import Any, List, Optional
+from typing import Any, Optional, Sequence
 
 from .scalar import Scalar
 
 
 class Function(Scalar):
     """Base function definition"""
 
-    _args: List[Any]
+    _args: Sequence[Any]
     _name: Optional[str] = None
 
     def __init__(self, *args: Any) -> None:
diff --git a/pysaql/scalar.py b/pysaql/scalar.py
@@ -1,11 +1,11 @@
 """Contains definition of a scalar"""
 
+from __future__ import annotations
+
 from abc import ABC, abstractmethod
 import operator
 from typing import Any, Callable, Optional, Sequence, Union
 
-from typing_extensions import Self
-
 from .expression import Expression
 from .util import escape_identifier, stringify
 
@@ -43,7 +43,7 @@ class Operation:
 class BooleanOperation(Operation):
     """Mixin that defines boolean comparison methods"""
 
-    def __and__(self, obj: Any) -> "BinaryOperation":
+    def __and__(self, obj: Any) -> BinaryOperation:
         """Creates a binary operation using the `and` operator
 
         Args:
@@ -55,7 +55,7 @@ def __and__(self, obj: Any) -> "BinaryOperation":
         """
         return BinaryOperation(operator.and_, self, obj)
 
-    def __or__(self, obj: Any) -> "BinaryOperation":
+    def __or__(self, obj: Any) -> BinaryOperation:
         """Creates a binary operation using the `or` operator
 
         Args:
@@ -67,7 +67,7 @@ def __or__(self, obj: Any) -> "BinaryOperation":
         """
         return BinaryOperation(operator.or_, self, obj, wrap=True)
 
-    def __invert__(self) -> "BinaryOperation":
+    def __invert__(self) -> UnaryOperation:
         """Creates a unary operation using the `inv` operator
 
         Returns:
@@ -134,7 +134,7 @@ class Scalar(Expression, BooleanOperation, ABC):
 
     _alias: Optional[str] = None
 
-    def alias(self, name: str) -> Self:
+    def alias(self, name: str) -> Scalar:
         """Set the alias name for a scalar expression
 
         Args:
@@ -165,7 +165,7 @@ def __str__(self) -> str:
 
         return s
 
-    def __eq__(self, obj: Any) -> BinaryOperation:
+    def __eq__(self, obj: Any) -> BinaryOperation:  # type: ignore[override]
         """Creates a binary operation using the `eq` or `is` operator
 
         Args:
@@ -178,7 +178,7 @@ def __eq__(self, obj: Any) -> BinaryOperation:
         op = operator.is_ if obj is None else operator.eq
         return BinaryOperation(op, self, obj)
 
-    def __ne__(self, obj: Any) -> BinaryOperation:
+    def __ne__(self, obj: Any) -> BinaryOperation:  # type: ignore[override]
         """Creates a binary operation using the `ne` or `is_not` operator
 
         Args:
diff --git a/pysaql/stream.py b/pysaql/stream.py
@@ -1,12 +1,12 @@
 """Contains stream expressions and statements"""
 
+from __future__ import annotations
+
 from abc import ABC
 import functools
 import operator
 from typing import List, Optional, Sequence, Tuple, Union
 
-from typing_extensions import Self
-
 from .enums import FillDateTypeString, JoinType, Order
 from .expression import Expression
 from .scalar import BinaryOperation, field, Scalar
@@ -83,7 +83,7 @@ def add_statement(self, statement: StreamStatement) -> None:
         """
         self._statements.append(statement)
 
-    def foreach(self, *fields: Scalar) -> Self:
+    def foreach(self, *fields: Scalar) -> Stream:
         """Applies a set of expressions to every row in a dataset.
 
         This action is often referred to as projection
@@ -98,14 +98,15 @@ def foreach(self, *fields: Scalar) -> Self:
         self._statements.append(ProjectionStatement(self, fields))
         return self
 
-    def group(self, *fields: Scalar) -> Self:
+    def group(self, *fields: Scalar) -> Stream:
         """Organizes the rows returned from a query into groups
 
         Within each group, you can apply an aggregate function, such as count() or sum()
         to get the number of items or sum, respectively.
 
         Args:
-            fields: One or more fields to group by
+            fields: One or more fields to group by. If no fields are provided,
+                "group by all" is assumed.
 
         Returns:
             self
@@ -114,7 +115,7 @@ def group(self, *fields: Scalar) -> Self:
         self._statements.append(GroupStatement(self, fields))
         return self
 
-    def filter(self, *filters: BinaryOperation) -> Self:
+    def filter(self, *filters: BinaryOperation) -> Stream:
         """Selects rows from a dataset based on a filter predicate
 
         Args:
@@ -128,7 +129,7 @@ def filter(self, *filters: BinaryOperation) -> Self:
         self._statements.append(FilterStatement(self, filters))
         return self
 
-    def order(self, *fields: Union[Scalar, Tuple[Scalar, Order]]) -> Self:
+    def order(self, *fields: Union[Scalar, Tuple[Scalar, Order]]) -> Stream:
         """Sorts in ascending or descending order on one or more fields.
 
         Args:
@@ -141,7 +142,7 @@ def order(self, *fields: Union[Scalar, Tuple[Scalar, Order]]) -> Self:
         self._statements.append(OrderStatement(self, fields))
         return self
 
-    def limit(self, limit: int) -> Self:
+    def limit(self, limit: int) -> Stream:
         """Limits the number of rows returned.
 
         Args:
@@ -159,7 +160,7 @@ def fill(
         date_cols: Sequence[field],
         date_type_string: FillDateTypeString,
         partition: Optional[field] = None,
-    ) -> Self:
+    ) -> Stream:
         """Fills missing date values by adding rows in data stream
 
         Args:
@@ -202,7 +203,7 @@ def __str__(self) -> str:
 class ProjectionStatement(StreamStatement):
     """Statement to project columns from a stream"""
 
-    def __init__(self, stream: Stream, fields: List[Scalar]) -> None:
+    def __init__(self, stream: Stream, fields: Sequence[Scalar]) -> None:
         """Initializer
 
         Args:
@@ -228,7 +229,7 @@ class OrderStatement(StreamStatement):
     def __init__(
         self,
         stream: Stream,
-        fields: Union[Scalar, List[Scalar], List[Tuple[Scalar, Order]]],
+        fields: Sequence[Union[Scalar, Tuple[Scalar, Order]]],
     ) -> None:
         """Initializer
 
@@ -284,30 +285,29 @@ def __str__(self) -> str:
 class GroupStatement(StreamStatement):
     """Statement to group rows in a stream"""
 
-    def __init__(self, stream: Stream, fields: List[Scalar]):
+    def __init__(self, stream: Stream, fields: Sequence[Scalar]):
         """Initializer
 
         Args:
             stream: Stream containing this statement
-            fields: One or more fields to group by
+            fields: One or more fields to group by. If no fields are provided,
+                "group by all" is assumed.
 
         """
         super().__init__()
         self.stream = stream
-        if not fields:
-            raise ValueError("At least one field is required")
         self.fields = fields
 
     def __str__(self) -> str:
         """Cast this group statement to a string"""
-        fields = ", ".join(str(f) for f in self.fields)
+        fields = stringify_list(self.fields) if self.fields else "all"
         return f"{self.stream.ref} = group {self.stream.ref} by {fields};"
 
 
 class FilterStatement(StreamStatement):
     """Statement to filter rows in a stream"""
 
-    def __init__(self, stream: Stream, filters: List[BinaryOperation]) -> None:
+    def __init__(self, stream: Stream, filters: Sequence[BinaryOperation]) -> None:
         """Initializer
 
         Args:
@@ -336,7 +336,7 @@ class CogroupStatement(StreamStatement):
     def __init__(
         self,
         stream: Stream,
-        streams: List[Tuple[Stream, Scalar]],
+        streams: Sequence[Tuple[Stream, Scalar]],
         join_type: JoinType = JoinType.inner,
     ) -> None:
         """Initializer
diff --git a/pysaql/util.py b/pysaql/util.py
@@ -1,7 +1,7 @@
 """Contains utility functions for working with expressions"""
 
 import json
-from typing import Any, List
+from typing import Any, Sequence
 
 
 def escape_identifier(s: str) -> str:
@@ -50,7 +50,7 @@ def stringify(obj: Any) -> str:
         return obj
 
 
-def stringify_list(seq: List) -> str:
+def stringify_list(seq: Sequence) -> str:
     """Cast a list into a string
 
     This will separate list items with commas and wrap the output in parenthesis if
@@ -63,5 +63,5 @@ def stringify_list(seq: List) -> str:
         stringified list
 
     """
-    seq = [seq] if not isinstance(seq, (list, tuple)) else seq
+    seq = [seq] if not isinstance(seq, (list, tuple, set)) else seq
     return f"({', '.join(str(s) for s in seq)})" if len(seq) > 1 else str(seq[0])
diff --git a/tests/unit/test_date.py b/tests/unit/test_date.py
@@ -1,8 +1,8 @@
 """Contains unit tests for the date_ module"""
 
-import pysaql.saql.date_ as mod_ut
-from pysaql.saql.enums import RelativeDateUnit, RelativeTimeframe
-from pysaql.saql.scalar import field
+import pysaql.date_ as mod_ut
+from pysaql.enums import RelativeDateUnit, RelativeTimeframe
+from pysaql.scalar import field
 
 
 def test_date():
diff --git a/tests/unit/test_query.py b/tests/unit/test_query.py
@@ -1,17 +1,17 @@
 """Contains tests rendered queries"""
 
-from pysaql.saql.aggregation import count, dense_rank, sum
-from pysaql.saql.date_ import date, date_range, day_in_week, relative_date, to_date
-from pysaql.saql.enums import (
+from pysaql.aggregation import count, dense_rank, sum
+from pysaql.date_ import date, date_range, day_in_week, relative_date, to_date
+from pysaql.enums import (
     FillDateTypeString,
     JoinType,
     Order,
     RelativeDateUnit,
     RelativeTimeframe,
 )
-from pysaql.saql.function import coalesce
-from pysaql.saql.scalar import field
-from pysaql.saql.stream import cogroup, load
+from pysaql.function import coalesce
+from pysaql.scalar import field
+from pysaql.stream import cogroup, load
 
 
 def test_complex():
diff --git a/tests/unit/test_scalar.py b/tests/unit/test_scalar.py
@@ -1,7 +1,7 @@
 """Contains unit tests for the scalar module"""
 
 
-from pysaql.saql.scalar import field
+from pysaql.scalar import field
 
 
 def test_alias():
diff --git a/tests/unit/test_stream.py b/tests/unit/test_stream.py
@@ -2,9 +2,9 @@
 
 import pytest
 
-from pysaql.saql.enums import FillDateTypeString, JoinType, Order
-from pysaql.saql.scalar import field
-from pysaql.saql.stream import cogroup, load, Stream
+from pysaql.enums import FillDateTypeString, JoinType, Order
+from pysaql.scalar import field
+from pysaql.stream import cogroup, load, Stream
 
 
 def test_load():
@@ -60,18 +60,18 @@ def test_foreach():
     assert str(stream) == "q0 = foreach q0 generate 'name', 'number' as 'n';"
 
 
-def test_group__invalid():
-    """Should raise when no fields provided"""
+def test_group__all():
+    """Should group by all when no fields are provided"""
     stream = Stream()
-    with pytest.raises(ValueError):
-        stream.group()
+    stream.group()
+    assert str(stream) == "q0 = group q0 by all;"
 
 
 def test_group():
     """Should group by fields"""
     stream = Stream()
     stream.group(field("name"), field("date"))
-    assert str(stream) == "q0 = group q0 by 'name', 'date';"
+    assert str(stream) == "q0 = group q0 by ('name', 'date');"
 
 
 def test_filter__invalid():
diff --git a/tests/unit/test_util.py b/tests/unit/test_util.py

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,3 @@`
`1`	`1`	`"""Python SAQL query builder"""`
`2`	`2`
`3`		`-__version__ = "0.2.0"`
	`3`	`+__version__ = "0.3.0"`