Add strongly typed filter

This makes it easy to write RSQL filters in python,
which then stringify back down to RSQL.

Author: Epse

docs/source/_templates/autosummary/class.rst

@@ -16,6 +16,7 @@
    {% for item in methods %}
    {% if item not in inherited_members %}
    .. automethod:: {{ item }}
+      :no-index-entry:
    {%- endif %}
    {%- endfor %}
    {% endif %}
@@ -29,6 +30,7 @@
    {% for item in attributes %}
    {% if item not in inherited_members %}
    .. autoattribute:: {{ item }}
+      :no-index-entry:
    {%- endif %}
    {%- endfor %}
    {% endif %}

docs/source/conf.py

@@ -28,6 +28,7 @@
     'sphinx.ext.autosummary',
     'sphinx.ext.viewcode',
     'sphinx.ext.doctest',
+    'sphinx.ext.intersphinx',
 ]
 autosummary_generate = True
 autosummary_ignore_module_all = False
@@ -49,3 +50,8 @@
 }
 autoclass_content = 'class'
 autodoc_member_order = 'groupwise'
+
+intersphinx_mapping = {
+    'python': ('https://docs.python.org/3', None),
+    'pydantic': ('https://docs.pydantic.dev/latest', None),
+}

src/obelisk/asynchronous/__init__.py

@@ -1,6 +1,6 @@
 """
 This module contains the asynchronous API to Obelisk-py.
-These methods all return a :any:`Coroutine`.
+These methods all return an :external+python:class:`collections.abc.Awaitable`.
 
 Relevant entrance points are :class:`client.Obelisk`.
 It can be imported from the :mod:`.client` module, or directly from this one.

src/obelisk/asynchronous/core.py

@@ -9,12 +9,13 @@
 """
 from obelisk.asynchronous.base import BaseClient
 from obelisk.exceptions import ObeliskError
+from obelisk.types.core import FieldName, Filter
 
 from datetime import datetime, timedelta
 import httpx
 import json
-from pydantic import BaseModel, AwareDatetime, ValidationError, model_validator
-from typing import AsyncIterator, Dict, Iterator, List, Literal, Optional, Any, get_args
+from pydantic import BaseModel, AwareDatetime, ConfigDict, Field, ValidationError, model_validator
+from typing import Annotated, AsyncIterator, Dict, Iterator, List, Literal, Optional, Any, get_args
 from typing_extensions import Self
 from numbers import Number
 
@@ -27,10 +28,6 @@
 """Type of aggregation Obelisk can process"""
 
 
-FieldName = str # TODO: validate field names?
-"""https://obelisk.pages.ilabt.imec.be/obelisk-core/query.html#available-data-point-fields"""
-
-
 Datapoint = Dict[str, Any]
 """Datapoints resulting from queries are modeled as simple dicts, as fields can come and go depending on query."""
 
@@ -93,10 +90,13 @@ class QueryParams(BaseModel):
     fields: Optional[List[FieldName]] = None
     orderBy: Optional[List[str]] = None # More complex than just FieldName, can be prefixed with - to invert sort
     dataType: Optional[DataType] = None
-    filter: Optional[str] = None # TODO: Validating this must be a nightmare
+    filter_: Annotated[Optional[str|Filter], Field(serialization_alias='filter')] = None
+    """Filter in `RSQL format <https://obelisk.pages.ilabt.imec.be/obelisk-core/query.html#rsql-format>`__ Suffix to avoid collisions."""
     cursor: Optional[str] = None
     limit: int = 1000
 
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
     @model_validator(mode='after')
     def check_datatype_needed(self) -> Self:
         if self.fields is None or 'value' in self.fields:
@@ -106,7 +106,7 @@ def check_datatype_needed(self) -> Self:
         return self
 
     def to_dict(self) -> Dict:
-        return self.model_dump(exclude_none=True)
+        return self.model_dump(exclude_none=True, by_alias=True)
 
 
 class ChunkedParams(BaseModel):
@@ -116,11 +116,14 @@ class ChunkedParams(BaseModel):
     fields: Optional[List[FieldName]] = None
     orderBy: Optional[List[str]] = None # More complex than just FieldName, can be prefixed with - to invert sort
     dataType: Optional[DataType] = None
-    filter: Optional[str] = None
+    filter_: Optional[str | Filter] = None
+    """Underscore suffix to avoid name collisions"""
     start: datetime
     end: datetime
     jump: timedelta = timedelta(hours=1)
 
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
     @model_validator(mode='after')
     def check_datatype_needed(self) -> Self:
         if self.fields is None or 'value' in self.fields:
@@ -133,9 +136,9 @@ def chunks(self) -> Iterator[QueryParams]:
         current_start = self.start
         while current_start < self.end:
             current_end = current_start + self.jump
-            filter=f'timestamp>={current_start.isoformat()};timestamp<{current_end.isoformat()}'
-            if self.filter:
-                filter += f';{self.filter}'
+            filter_=f'timestamp>={current_start.isoformat()};timestamp<{current_end.isoformat()}'
+            if self.filter_:
+                filter_ += f';{self.filter_}'
 
             yield QueryParams(
                 dataset=self.dataset,
@@ -144,7 +147,7 @@ def chunks(self) -> Iterator[QueryParams]:
                 fields=self.fields,
                 orderBy=self.orderBy,
                 dataType=self.dataType,
-                filter=filter
+                filter_=filter_
             )
 
             current_start += self.jump

src/obelisk/types.py renamed to src/obelisk/types/__init__.py

@@ -0,0 +1,129 @@
+from __future__ import annotations
+from abc import ABC
+from datetime import datetime
+from typing import Any, Iterable, List
+
+
+FieldName = str # TODO: validate field names?
+"""https://obelisk.pages.ilabt.imec.be/obelisk-core/query.html#available-data-point-fields"""
+
+
+class Constraint(ABC):
+    pass
+
+
+class Comparison():
+    left: FieldName
+    right: Any
+    op: str
+
+    def __init__(self, left: FieldName, right: Any, op: str):
+        self.left = left
+        self.right = right
+        self.op = op
+
+    def __str__(self) -> str:
+        right = self._sstr(self.right)
+        if not right.startswith('('):
+            right = f"'{right}'"
+
+        return f"('{self.left}'{self.op}{right})"
+
+    @staticmethod
+    def _sstr(item: Any):
+        """Smart string conversion"""
+        if isinstance(item, datetime):
+            return item.isoformat()
+        return str(item)
+
+    @staticmethod
+    def _iterable_to_group(iter: Iterable[Any]) -> str:
+        """Produces a group of the form ("a","b")"""
+        return str(tuple([Comparison._sstr(x) for x in iter]))
+
+    @classmethod
+    def equal(cls, left: FieldName, right: Any) -> Comparison:
+        return Comparison(left, right, "==")
+
+    @classmethod
+    def not_equal(cls, left: FieldName, right: Any) -> Comparison:
+        return Comparison(left, right, "!=")
+
+    @classmethod
+    def less(cls, left: FieldName, right: Any) -> Comparison:
+        return Comparison(left, right, "<")
+
+    @classmethod
+    def less_equal(cls, left: FieldName, right: Any) -> Comparison:
+        return Comparison(left, right, "<=")
+
+    @classmethod
+    def greater(cls, left: FieldName, right: Any) -> Comparison:
+        return Comparison(left, right, ">")
+
+    @classmethod
+    def greater_equal(cls, left: FieldName, right: Any) -> Comparison:
+        return Comparison(left, right, ">=")
+
+    @classmethod
+    def is_in(cls, left: FieldName, right: Iterable[Any]) -> Comparison:
+        return Comparison(left, cls._iterable_to_group(right), "=in=")
+
+    @classmethod
+    def is_not_in(cls, left: FieldName, right: Iterable[Any]) -> Comparison:
+        return Comparison(left, cls._iterable_to_group(right), "=out=")
+
+    @classmethod
+    def null(cls, left: FieldName) -> Comparison:
+        return Comparison(left, "", "=null=")
+
+    @classmethod
+    def not_null(cls, left: FieldName) -> Comparison:
+        return Comparison(left, "", "=notnull=")
+
+
+Item = Constraint | Comparison
+
+
+class And(Constraint):
+    content: List[Item]
+
+    def __init__(self, *args: Item):
+        self.content = list(args)
+
+    def __str__(self) -> str:
+        return "(" + ";".join([str(x) for x in self.content]) + ")"
+
+
+class Or(Constraint):
+    content: List[Item]
+
+    def __init__(self, *args: Item):
+        self.content = list(args)
+
+    def __str__(self) -> str:
+        return "(" + ",".join([str(x) for x in self.content]) + ")"
+
+
+class Filter():
+    content: Item | None = None
+
+    def __init__(self, content: Constraint | None = None):
+        self.content = content
+
+    def __str__(self) -> str:
+        return str(self.content)
+
+    def add_and(self, *other: Item) -> Filter:
+        if self.content is None:
+            self.content = And(*other)
+        else:
+            self.content = And(self.content, *other)
+        return self
+
+    def add_or(self, *other: Item) -> Filter:
+        if self.content is None:
+            self.content = Or(*other)
+        else:
+            self.content = Or(self.content, *other)
+        return self

src/obelisk/types/core.py

@@ -0,0 +1,6 @@
+from obelisk.asynchronous.core import QueryParams
+
+def test_query_param_serialize():
+    q = QueryParams(dataset="83989232", filter_="(metric=='smartphone.application::string')", dataType='string')
+    dump = q.to_dict()
+    assert "filter" in dump

src/tests/asynchronous/core_test.py

@@ -0,0 +1,19 @@
+from obelisk.types.core import Filter, Comparison
+from datetime import datetime
+
+
+def test_basic_filter():
+    test_dt = datetime.now()
+    f = Filter() \
+        .add_and(
+            Comparison.equal('source', 'test source'),
+        )\
+        .add_or(
+            Comparison.less('timestamp', test_dt)
+        )\
+        .add_or(
+            Comparison.is_in('metricType', ['number', 'number[]']),
+        )
+
+    expected = f"(((('source'=='test source')),('timestamp'<'{test_dt.isoformat()}')),('metricType'=in=('number', 'number[]')))"
+    assert str(f) == expected