more lenient parsing for inconsistent event data

Author: tradel

CHANGELOG.md

@@ -1,5 +1,11 @@
 # PyBanyan Change History
 
+## 0.15.0
+ * allow for inconsistent timestamps in event data
+ * implement paging for events v2 API
+ * fix broken callable support in API base class
+ * more parser tests for events
+
 ## 0.14.0
  * more updates to events model 
  * added filter to remove null fields in JSON output

banyan/api/__init__.py

@@ -198,7 +198,7 @@ def paged_request(self, method: str, uri: str, params: Dict[str, Any] = None, da
             results = self.api_request(method, uri, params, data, json, headers, accept)
             for key in results.keys():
                 logging.debug(f'Looking for {key} in {uri}')
-                if key in uri:
+                if key in uri or key == 'data':
                     if len(results[key]) == 0:
                         return all_results
                     all_results.extend(results[key])

banyan/api/base.py

@@ -1,5 +1,5 @@
 from abc import ABC
-from typing import Dict, List, Union, Iterable, Any
+from typing import Dict, List, Union, Iterable, Any, Callable
 
 from banyan.core.exc import BanyanError
 from banyan.model import InfoBase, BanyanApiObject, Resource, ResourceOrName
@@ -73,7 +73,7 @@ def args_to_html_params(args: Iterable) -> Dict[str, Any]:
         params = dict()
         for arg, key, val in args:
             if arg:
-                params[key] = val
+                params[key] = val() if isinstance(val, Callable) else val
         return params
 
     def find(self, obj: ResourceOrName):

banyan/api/event_v2.py

@@ -14,7 +14,7 @@ class Meta:
         list_uri = '/events'
         uri_param = 'EventID'
         obj_name = 'event'
-        supports_paging = False
+        supports_paging = True
 
     def create(self, obj: BanyanApiObject) -> str:
         raise NotImplementedError('The Banyan API does not support this operation')
@@ -43,8 +43,7 @@ def list(self, before_dt: datetime = None, after_dt: datetime = None, order: str
             (service_name, 'service_name', service_name),
             (event_id, 'id', event_id),
         ])
-        response_json = self._client.api_request('GET', self.Meta.list_uri, params=params)
-        response_json = response_json['data']
+        response_json = self._client.paged_request('GET', self.Meta.list_uri, params=params)
         data: List[Resource] = self.Meta.info_class.Schema().load(response_json, many=True)
         self._build_cache(data)
         return data

banyan/core/version.py

@@ -1,7 +1,7 @@
 
 from cement.utils.version import get_version as cement_get_version
 
-VERSION = (0, 14, 0, 'final', 1)
+VERSION = (0, 15, 0, 'final', 1)
 
 
 def get_version(version=VERSION):

banyan/model/__init__.py

@@ -29,8 +29,8 @@ def _deserialize(self, value, attr, data, **kwargs):
         elif isinstance(value, int):
             try:
                 return datetime.fromtimestamp(value)
-            except Exception as ex:
-                raise ValueError(f'{ex.args[0]}, value = {value}, attr = {attr}')
+            except OSError:
+                return None
         else:
             return super()._deserialize(value, attr, data, **kwargs)
 
@@ -59,7 +59,10 @@ def _serialize(self, value: datetime, attr, obj, **kwargs):
         if not value:
             return 0
         elif isinstance(value, datetime):
-            return int(value.timestamp() * 1000000000)
+            try:
+                return int(value.timestamp() * 1000000000)
+            except OSError:
+                return 0
         else:
             return super()._serialize(value, attr, obj, **kwargs)
 

banyan/model/event_v2.py

@@ -7,7 +7,7 @@
 from marshmallow.fields import String
 from marshmallow_dataclass import dataclass
 
-from banyan.model import Resource, NanoTimestampField, MilliTimestampField, BanyanEnum
+from banyan.model import Resource, NanoTimestampField, MilliTimestampField, BanyanEnum, TimestampField
 
 
 @dataclass
@@ -34,7 +34,7 @@ class EventDevice:
     class Meta:
         unknown = EXCLUDE
 
-    device_id: Optional[UUID] = field(metadata={"data_key": "id"})
+    device_id: str = field(metadata={"data_key": "id"})
     friendly_name: str
     mac_address: str
     serial_number: str
@@ -48,14 +48,8 @@ class Meta:
     architecture: str
     udid: str
     source: str
-    last_mdm_data_synced_at: datetime = field(metadata={'marshmallow_field': MilliTimestampField()})
-
-    # noinspection PyUnusedLocal
-    @pre_load
-    def _remove_nulls(self, data, many, **kwargs):
-        if "id" in data and not data["id"]:
-            del data["id"]
-        return data
+    last_mdm_data_synced_at: datetime = field(metadata={'marshmallow_field': TimestampField()})
+    last_user_agent: Optional[str]
 
 
 @dataclass
@@ -72,7 +66,7 @@ class EventUserPrincipal:
     class Meta:
         unknown = EXCLUDE
 
-    user: EventUser
+    user: Optional[EventUser]
     device: Optional[EventDevice]
     client: Optional[EventClient]