Skip to content

Commit 94e6a6b

Browse files
author
Todd Radel
authored
Merge pull request #20 from banyansecurity/null-output-filter
remove nulls from json output
2 parents c4bd240 + 7075d70 commit 94e6a6b

File tree

8 files changed

+639
-35
lines changed

8 files changed

+639
-35
lines changed

‎CHANGELOG.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
# PyBanyan Change History
22

3+
## 0.14.0
4+
* more updates to events model
5+
* added filter to remove null fields in JSON output
6+
* Pythonified field names in EventV2TypeCount
7+
38
## 0.13.0
49
* updated events model to latest v2 API
510

‎banyan/core/version.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11

22
from cement.utils.version import get_version as cement_get_version
33

4-
VERSION = (0, 13, 0, 'final', 1)
4+
VERSION = (0, 14, 0, 'final', 1)
55

66

77
def get_version(version=VERSION):

‎banyan/model/__init__.py‎

Lines changed: 29 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
from typing import ClassVar, Type, Union
66

77
from aenum import StrEnum
8-
from marshmallow import fields, Schema
8+
from marshmallow import fields, Schema, post_dump
99
from marshmallow_dataclass import dataclass
1010
from semver import VersionInfo
1111

@@ -27,7 +27,10 @@ def _deserialize(self, value, attr, data, **kwargs):
2727
if not value:
2828
return None
2929
elif isinstance(value, int):
30-
return datetime.fromtimestamp(value)
30+
try:
31+
return datetime.fromtimestamp(value)
32+
except Exception as ex:
33+
raise ValueError(f'{ex.args[0]}, value = {value}, attr = {attr}')
3134
else:
3235
return super()._deserialize(value, attr, data, **kwargs)
3336

@@ -45,7 +48,10 @@ def _deserialize(self, value, attr, data, **kwargs):
4548
if not value:
4649
return None
4750
elif isinstance(value, int):
48-
return datetime.fromtimestamp(value / 1000000000)
51+
try:
52+
return datetime.fromtimestamp(value / 1000000000)
53+
except Exception as ex:
54+
raise ValueError(f'{ex.args[0]}, value = {value}, attr = {attr}')
4955
else:
5056
return super()._deserialize(value, attr, data, **kwargs)
5157

@@ -65,7 +71,7 @@ def _deserialize(self, value, attr, data, **kwargs):
6571
elif isinstance(value, int):
6672
try:
6773
return datetime.fromtimestamp(value / 1000)
68-
except ValueError as ex:
74+
except Exception as ex:
6975
raise ValueError(f'{ex.args[0]}, value = {value}, attr = {attr}')
7076
else:
7177
return super()._deserialize(value, attr, data, **kwargs)
@@ -84,7 +90,10 @@ def _deserialize(self, value, attr, data, **kwargs):
8490
if not value:
8591
return None
8692
elif isinstance(value, str):
87-
return IPv4Interface(value)
93+
try:
94+
return IPv4Interface(value)
95+
except Exception as ex:
96+
raise ValueError(f'{ex.args[0]}, value = {value}, attr = {attr}')
8897
else:
8998
return super()._deserialize(value, attr, data, **kwargs)
9099

@@ -104,7 +113,10 @@ def _deserialize(self, value, attr, data, **kwargs):
104113
elif isinstance(value, str):
105114
if '_' in value:
106115
value = value.split('_')[0]
107-
return VersionInfo.parse(value)
116+
try:
117+
return VersionInfo.parse(value)
118+
except Exception as ex:
119+
raise ValueError(f'{ex.args[0]}, value = {value}, attr = {attr}')
108120
else:
109121
return super()._deserialize(value, attr, data, **kwargs)
110122

@@ -146,6 +158,17 @@ def name(self) -> str:
146158
def id(self) -> str:
147159
raise BanyanError('not implemented')
148160

161+
# noinspection PyUnusedLocal
162+
@post_dump
163+
def _remove_nulls(self, data, many, **kwargs):
164+
new_data = dict()
165+
for key, val in data.items():
166+
if isinstance(val, dict):
167+
new_data[key] = self._remove_nulls(val, False, **kwargs)
168+
elif val is not None:
169+
new_data[key] = val
170+
return new_data
171+
149172

150173
ResourceOrName = Union[Resource, str]
151174

‎banyan/model/event_v2.py‎

Lines changed: 31 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,15 @@ class Meta:
1919
groups: List[str] = field(default_factory=list)
2020
roles: List[str] = field(default_factory=list)
2121

22+
# noinspection PyUnusedLocal
23+
@pre_load
24+
def _remove_nulls(self, data, many, **kwargs):
25+
if "roles" in data and data["roles"] is None:
26+
del data["roles"]
27+
if "groups" in data and data["groups"] is None:
28+
del data["groups"]
29+
return data
30+
2231

2332
@dataclass
2433
class EventDevice:
@@ -41,6 +50,13 @@ class Meta:
4150
source: str
4251
last_mdm_data_synced_at: datetime = field(metadata={'marshmallow_field': MilliTimestampField()})
4352

53+
# noinspection PyUnusedLocal
54+
@pre_load
55+
def _remove_nulls(self, data, many, **kwargs):
56+
if "id" in data and not data["id"]:
57+
del data["id"]
58+
return data
59+
4460

4561
@dataclass
4662
class EventClient:
@@ -57,7 +73,7 @@ class Meta:
5773
unknown = EXCLUDE
5874

5975
user: EventUser
60-
device: EventDevice
76+
device: Optional[EventDevice]
6177
client: Optional[EventClient]
6278

6379

@@ -105,7 +121,7 @@ class Meta:
105121
role_name: str = field(metadata={"data_key": "name"})
106122
version: int
107123
bound_by: str
108-
bound_at: datetime = field(metadata={'marshmallow_field': MilliTimestampField()})
124+
bound_at: datetime = field(metadata={'marshmallow_field': NanoTimestampField()})
109125

110126

111127
@dataclass
@@ -193,7 +209,7 @@ class Meta:
193209
container_name: str
194210
service_id: str
195211
service_name: str
196-
service_version: Optional[int] = field(metadata={"marshmallow_field": String(data_key="serivce_version")})
212+
service_version: Optional[int] = field(metadata={"marshmallow_field": String()})
197213
host_name: str
198214
ip: str
199215
port: Optional[str]
@@ -230,12 +246,12 @@ class EventV2Type(BanyanEnum):
230246
ACCESS = "Access"
231247
IDENTITY = "Identity"
232248
REGISTRATION = "Registration"
233-
TRUSTSCORING = "TrustScoring"
249+
TRUST_SCORING = "TrustScoring"
234250

235251

236252
class EventV2Subtype(BanyanEnum):
237253
DEVICE = "Device"
238-
USERPRINCIPAL = "UserPrincipal"
254+
USER_PRINCIPAL = "UserPrincipal"
239255
CONNECTION = "Connection"
240256
RESOURCE = "Resource"
241257

@@ -274,31 +290,22 @@ def name(self) -> str:
274290
def id(self) -> str:
275291
return str(self.event_id)
276292

277-
# noinspection PyUnusedLocal
278-
@pre_load
279-
def _remove_empty_fields(self, data, many, **kwargs):
280-
if data["user_principal"]["device"]["id"] == "":
281-
del data["user_principal"]["device"]["id"]
282-
if data["user_principal"]["user"]["roles"] is None:
283-
data["user_principal"]["user"]["roles"] = list()
284-
return data
285-
286293

287294
@dataclass
288295
class EventV2TypeCount(Resource):
289296
class Meta:
290297
unknown = EXCLUDE
291298

292-
StatsEndTime: datetime
293-
DeltaTime: int
294-
NumEventTypeAccessAuth: int
295-
NumEventTypeAccessUnauth: int
296-
NumEventTypeIdentityDeny: int
297-
NumEventTypeService: int
298-
NumEventTypeIdentityGrant: int
299-
NumEventTypeLink: int
300-
NumEventTypePolicy: int
301-
NumEventTypeRole: int
299+
end_date: datetime = field(metadata={"data_key": "StatsEndTime"})
300+
delta_time: int = field(metadata={"data_key": "DeltaTime"})
301+
access_auth: int = field(metadata={"data_key": "NumEventTypeAccessAuth"})
302+
access_unauth: int = field(metadata={"data_key": "NumEventTypeAccessUnauth"})
303+
identity_grant: int = field(metadata={"data_key": "NumEventTypeIdentityGrant"})
304+
identity_deny: int = field(metadata={"data_key": "NumEventTypeIdentityDeny"})
305+
service: int = field(metadata={"data_key": "NumEventTypeService"})
306+
link: int = field(metadata={"data_key": "NumEventTypeLink"})
307+
policy: int = field(metadata={"data_key": "NumEventTypePolicy"})
308+
role: int = field(metadata={"data_key": "NumEventTypeRole"})
302309

303310

304311
EventOrID = Union[EventV2, str]

‎tests/data/event_v2.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@
127127
"container_name": "",
128128
"service_id": "dsrini-edge-plain-web-service.shaobo.bnn",
129129
"service_name": "dsrini-edge-plain-web-service.shaobo.bnn",
130-
"serivce_version": "",
130+
"service_version": "",
131131
"host_name": "",
132132
"ip": "10.128.0.25:80",
133133
"port": ""

‎tests/data/events_v2.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@
128128
"container_name": "",
129129
"service_id": "dsrini-edge-plain-web-service.shaobo.bnn",
130130
"service_name": "dsrini-edge-plain-web-service.shaobo.bnn",
131-
"serivce_version": "",
131+
"service_version": "",
132132
"host_name": "",
133133
"ip": "10.128.0.25:80",
134134
"port": ""
@@ -264,7 +264,7 @@
264264
"container_name": "",
265265
"service_id": "dsrini-edge-web-service.shaobo.bnn",
266266
"service_name": "dsrini-edge-web-service.shaobo.bnn",
267-
"serivce_version": "",
267+
"service_version": "",
268268
"host_name": "",
269269
"ip": "34.70.156.134:443",
270270
"port": ""
@@ -400,7 +400,7 @@
400400
"container_name": "",
401401
"service_id": "dsrini-edge-web-service.shaobo.bnn",
402402
"service_name": "dsrini-edge-web-service.shaobo.bnn",
403-
"serivce_version": "",
403+
"service_version": "",
404404
"host_name": "",
405405
"ip": "34.70.156.134:80",
406406
"port": ""

0 commit comments

Comments
 (0)