Skip to content

Commit b61eb4e

Browse files
fabiocaccamofabiocaccamo
committed
Improved csv I/O support and refacored io dict and utils.
1 parent 2f1f680 commit b61eb4e

File tree

7 files changed

+484
-185
lines changed

7 files changed

+484
-185
lines changed

‎benedict/dicts/__init__.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,8 +52,13 @@ def fromkeys(cls, sequence, value=None):
5252

5353
@staticmethod
5454
@benediction
55-
def from_base64(s, format='json', **kwargs):
56-
return IODict.from_base64(s, format, **kwargs)
55+
def from_base64(s, subformat='json', encoding='utf-8', **kwargs):
56+
return IODict.from_base64(s, subformat=subformat, encoding=encoding, **kwargs)
57+
58+
@staticmethod
59+
@benediction
60+
def from_csv(s, columns=None, columns_row=True, **kwargs):
61+
return IODict.from_csv(s, columns=columns, columns_row=columns_row, **kwargs)
5762

5863
@staticmethod
5964
@benediction

‎benedict/dicts/io.py‎

Lines changed: 54 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -8,30 +8,34 @@
88
class IODict(dict):
99

1010
def __init__(self, *args, **kwargs):
11-
# if first argument is data-string,
12-
# try to decode it using all decoders.
11+
# if first argument is data-string try to decode it.
12+
# use 'format' kwarg to specify the decoder to use, default 'json'.
1313
if len(args) and isinstance(args[0], string_types):
14-
d = IODict._from_any_data_string(args[0], **kwargs)
14+
s = args[0]
15+
format = kwargs.pop('format', 'json').lower()
16+
if format in ['b64', 'base64']:
17+
kwargs.setdefault('subformat', 'json')
18+
# decode data-string and initialize with dict data.
19+
d = IODict._decode(s, format, **kwargs)
1520
if d and isinstance(d, dict):
16-
args = list(args)
17-
args[0] = d
18-
args = tuple(args)
21+
super(IODict, self).__init__(d)
1922
else:
2023
raise ValueError('Invalid string data input.')
21-
super(IODict, self).__init__(*args, **kwargs)
24+
else:
25+
super(IODict, self).__init__(*args, **kwargs)
2226

2327
@staticmethod
24-
def _decode(s, decoder, **kwargs):
28+
def _decode(s, format, **kwargs):
2529
d = None
2630
try:
2731
content = io_util.read_content(s)
28-
# decode content using the given decoder
29-
data = decoder(content, **kwargs)
32+
# decode content using the given format
33+
data = io_util.decode(content, format, **kwargs)
3034
if isinstance(data, dict):
3135
d = data
3236
elif isinstance(data, list):
3337
# force list to dict
34-
d = { 'values':data }
38+
d = { 'values': data }
3539
else:
3640
raise ValueError(
3741
'Invalid data type: {}, expected dict or list.'.format(type(data)))
@@ -41,90 +45,66 @@ def _decode(s, decoder, **kwargs):
4145
return d
4246

4347
@staticmethod
44-
def _encode(d, encoder, filepath=None, **kwargs):
45-
s = encoder(d, **kwargs)
48+
def _encode(d, format, **kwargs):
49+
filepath = kwargs.pop('filepath', None)
50+
s = io_util.encode(d, format, **kwargs)
4651
if filepath:
4752
io_util.write_file(filepath, s)
4853
return s
4954

5055
@staticmethod
51-
def _from_any_data_string(s, **kwargs):
52-
funcs = [
53-
IODict.from_base64,
54-
IODict.from_json,
55-
IODict.from_query_string,
56-
IODict.from_toml,
57-
IODict.from_xml,
58-
IODict.from_yaml,
59-
]
60-
for f in funcs:
61-
try:
62-
options = kwargs.copy()
63-
d = f(s, **options)
64-
return d
65-
except ValueError:
66-
pass
56+
def from_base64(s, subformat='json', encoding='utf-8', **kwargs):
57+
kwargs['subformat'] = subformat
58+
kwargs['encoding'] = encoding
59+
return IODict._decode(s, 'base64', **kwargs)
6760

6861
@staticmethod
69-
def from_base64(s, format='json', encoding='utf-8', **kwargs):
70-
kwargs['format'] = format
71-
kwargs['encoding'] = encoding
72-
return IODict._decode(s,
73-
decoder=io_util.decode_base64, **kwargs)
62+
def from_csv(s, columns=None, columns_row=True, **kwargs):
63+
kwargs['columns'] = columns
64+
kwargs['columns_row'] = columns_row
65+
return IODict._decode(s, 'csv', **kwargs)
7466

7567
@staticmethod
7668
def from_json(s, **kwargs):
77-
return IODict._decode(s,
78-
decoder=io_util.decode_json, **kwargs)
69+
return IODict._decode(s, 'json', **kwargs)
7970

8071
@staticmethod
8172
def from_query_string(s, **kwargs):
82-
return IODict._decode(s,
83-
decoder=io_util.decode_query_string, **kwargs)
73+
return IODict._decode(s, 'query_string', **kwargs)
8474

8575
@staticmethod
8676
def from_toml(s, **kwargs):
87-
return IODict._decode(s,
88-
decoder=io_util.decode_toml, **kwargs)
77+
return IODict._decode(s, 'toml', **kwargs)
8978

9079
@staticmethod
9180
def from_xml(s, **kwargs):
92-
return IODict._decode(s,
93-
decoder=io_util.decode_xml, **kwargs)
81+
return IODict._decode(s, 'xml', **kwargs)
9482

9583
@staticmethod
9684
def from_yaml(s, **kwargs):
97-
return IODict._decode(s,
98-
decoder=io_util.decode_yaml, **kwargs)
85+
return IODict._decode(s, 'yaml', **kwargs)
9986

100-
def to_base64(self, filepath=None, format='json', encoding='utf-8', **kwargs):
101-
kwargs['format'] = format
87+
def to_base64(self, subformat='json', encoding='utf-8', **kwargs):
88+
kwargs['subformat'] = subformat
10289
kwargs['encoding'] = encoding
103-
return IODict._encode(self,
104-
encoder=io_util.encode_base64,
105-
filepath=filepath, **kwargs)
106-
107-
def to_json(self, filepath=None, **kwargs):
108-
return IODict._encode(self,
109-
encoder=io_util.encode_json,
110-
filepath=filepath, **kwargs)
111-
112-
def to_query_string(self, filepath=None, **kwargs):
113-
return IODict._encode(self,
114-
encoder=io_util.encode_query_string,
115-
filepath=filepath, **kwargs)
116-
117-
def to_toml(self, filepath=None, **kwargs):
118-
return IODict._encode(self,
119-
encoder=io_util.encode_toml,
120-
filepath=filepath, **kwargs)
121-
122-
def to_xml(self, filepath=None, **kwargs):
123-
return IODict._encode(self,
124-
encoder=io_util.encode_xml,
125-
filepath=filepath, **kwargs)
126-
127-
def to_yaml(self, filepath=None, **kwargs):
128-
return IODict._encode(self,
129-
encoder=io_util.encode_yaml,
130-
filepath=filepath, **kwargs)
90+
return IODict._encode(self, 'base64', **kwargs)
91+
92+
def to_csv(self, key='values', columns=None, columns_row=True, **kwargs):
93+
kwargs['columns'] = columns
94+
kwargs['columns_row'] = columns_row
95+
return IODict._encode(self[key], 'csv', **kwargs)
96+
97+
def to_json(self, **kwargs):
98+
return IODict._encode(self, 'json', **kwargs)
99+
100+
def to_query_string(self, **kwargs):
101+
return IODict._encode(self, 'query_string', **kwargs)
102+
103+
def to_toml(self, **kwargs):
104+
return IODict._encode(self, 'toml', **kwargs)
105+
106+
def to_xml(self, **kwargs):
107+
return IODict._encode(self, 'xml', **kwargs)
108+
109+
def to_yaml(self, **kwargs):
110+
return IODict._encode(self, 'yaml', **kwargs)

‎benedict/utils/io_util.py‎

Lines changed: 85 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# -*- coding: utf-8 -*-
22

33
from six import binary_type, string_types, StringIO
4+
from slugify import slugify
45

56
import base64
67
import csv
@@ -27,6 +28,16 @@
2728
from urlparse import parse_qs
2829

2930

31+
def decode(s, format, **kwargs):
32+
decode_func = _get_format_decoder(format)
33+
if decode_func:
34+
decode_opts = kwargs.copy()
35+
data = decode_func(s.strip(), **decode_opts)
36+
return data
37+
else:
38+
raise ValueError('Invalid format: {}.'.format(format))
39+
40+
3041
def decode_base64(s, **kwargs):
3142
# fix urlencoded chars
3243
s = unquote(s)
@@ -35,18 +46,12 @@ def decode_base64(s, **kwargs):
3546
if m != 0:
3647
s += '=' * (4 - m)
3748
data = base64.b64decode(s)
38-
format = kwargs.pop('format', None)
39-
encoding = kwargs.pop('encoding', 'utf-8' if format else None)
49+
subformat = kwargs.pop('subformat', None)
50+
encoding = kwargs.pop('encoding', 'utf-8' if subformat else None)
4051
if encoding:
4152
data = data.decode(encoding)
42-
if format:
43-
decoders = {
44-
'json': decode_json,
45-
'toml': decode_toml,
46-
'yaml': decode_yaml,
47-
'xml': decode_xml,
48-
}
49-
decode_func = decoders.get(format.lower(), '')
53+
if subformat:
54+
decode_func = _get_format_decoder(subformat)
5055
if decode_func:
5156
data = decode_func(data, **kwargs)
5257
return data
@@ -109,18 +114,21 @@ def decode_yaml(s, **kwargs):
109114
return data
110115

111116

117+
def encode(d, format, **kwargs):
118+
encode_func = _get_format_encoder(format)
119+
if encode_func:
120+
s = encode_func(d, **kwargs)
121+
return s
122+
else:
123+
raise ValueError('Invalid format: {}.'.format(format))
124+
125+
112126
def encode_base64(d, **kwargs):
113127
data = d
114-
format = kwargs.pop('format', None)
115-
encoding = kwargs.pop('encoding', 'utf-8' if format else None)
116-
if not isinstance(data, string_types) and format:
117-
encoders = {
118-
'json': encode_json,
119-
'toml': encode_toml,
120-
'yaml': encode_yaml,
121-
'xml': encode_xml,
122-
}
123-
encode_func = encoders.get(format.lower(), '')
128+
subformat = kwargs.pop('subformat', None)
129+
encoding = kwargs.pop('encoding', 'utf-8' if subformat else None)
130+
if not isinstance(data, string_types) and subformat:
131+
encode_func = _get_format_encoder(subformat)
124132
if encode_func:
125133
data = encode_func(data, **kwargs)
126134
if isinstance(data, string_types) and encoding:
@@ -225,3 +233,60 @@ def write_file(filepath, content):
225233
handler.write(content)
226234
handler.close()
227235
return True
236+
237+
238+
_formats = {
239+
'b64': {
240+
'decoder': decode_base64,
241+
'encoder': encode_base64,
242+
},
243+
'base64': {
244+
'decoder': decode_base64,
245+
'encoder': encode_base64,
246+
},
247+
'csv': {
248+
'decoder': decode_csv,
249+
'encoder': encode_csv,
250+
},
251+
'json': {
252+
'decoder': decode_json,
253+
'encoder': encode_json,
254+
},
255+
'qs': {
256+
'decoder': decode_query_string,
257+
'encoder': encode_query_string,
258+
},
259+
'query_string': {
260+
'decoder': decode_query_string,
261+
'encoder': encode_query_string,
262+
},
263+
'toml': {
264+
'decoder': decode_toml,
265+
'encoder': encode_toml,
266+
},
267+
'yaml': {
268+
'decoder': decode_yaml,
269+
'encoder': encode_yaml,
270+
},
271+
'yml': {
272+
'decoder': decode_yaml,
273+
'encoder': encode_yaml,
274+
},
275+
'xml': {
276+
'decoder': decode_xml,
277+
'encoder': encode_xml,
278+
},
279+
}
280+
281+
282+
def _get_format(format):
283+
return _formats.get(
284+
slugify(format, separator='_'), {})
285+
286+
287+
def _get_format_decoder(format):
288+
return _get_format(format).get('decoder', None)
289+
290+
291+
def _get_format_encoder(format):
292+
return _get_format(format).get('encoder', None)

‎tests/input/invalid-content.csv‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Lorem ipsum consectetur sint id aute officia sed excepteur consectetur labore laboris dolore in labore consequat ut in eu ut deserunt.
2+
Elit aliqua velit aliquip voluptate consequat reprehenderit occaecat dolor ut esse aute laboris cillum fugiat esse est laborum.

‎tests/input/valid-content.csv‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
id,name,age,height,weight
2+
1,Alice,20,62,120.6
3+
2,Freddie,21,74,190.6
4+
3,Bob,17,68,120.0
5+
4,François,32,75,110.05

0 commit comments

Comments
 (0)