Skip to content

Commit ddab733

Browse files
author
Paul Santa Clara
committed
parse time_extracted with faster ciso8601 lib. parse line json loads numbers as decimals
1 parent 6118a61 commit ddab733

File tree

2 files changed

+13
-2
lines changed

2 files changed

+13
-2
lines changed

setup.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
'simplejson==3.11.1',
1616
'python-dateutil>=2.6.0',
1717
'backoff==1.8.0',
18+
'ciso8601',
1819
],
1920
extras_require={
2021
'dev': [

singer/messages.py

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@
55
import simplejson as json
66

77
import singer.utils as u
8+
import ciso8601
9+
from .logger import get_logger
10+
LOGGER = get_logger()
811

912
class Message():
1013
'''Base class for messages.'''
@@ -180,13 +183,20 @@ def parse_message(msg):
180183
# lossy conversions. However, this will affect
181184
# very few data points and we have chosen to
182185
# leave conversion as is for now.
183-
obj = json.loads(msg)
186+
obj = json.loads(msg, use_decimal=True)
184187
msg_type = _required_key(obj, 'type')
185188

186189
if msg_type == 'RECORD':
187190
time_extracted = obj.get('time_extracted')
188191
if time_extracted:
189-
time_extracted = dateutil.parser.parse(time_extracted)
192+
try:
193+
time_extracted = ciso8601.parse_datetime(time_extracted)
194+
except:
195+
LOGGER.warning("unable to parse time_extracted with ciso8601 library")
196+
time_extracted = None
197+
198+
199+
# time_extracted = dateutil.parser.parse(time_extracted)
190200
return RecordMessage(stream=_required_key(obj, 'stream'),
191201
record=_required_key(obj, 'record'),
192202
version=obj.get('version'),

0 commit comments

Comments
 (0)