Skip to content

Commit cf4346e

Browse files
chrisrink10chrisrink10
authored
Add JSON encoder and decoder (#485)
Add a JSON encoder and decoder based on Python's `json` module. This is obviously not going to be the fastest encoder or decoder in the planet, but it will be builtin so folks don't have to fetch another library to do the job. It's made worse by the fact that Python's builtin decoder does not include an `array_hook` option. We could probably make a fairly efficient decoder with that and the existing `object_hook`, but since we have the completely decode the entire object to Python objects and then convert, it's probably pretty slow.
1 parent 0b93497 commit cf4346e

File tree

6 files changed

+496
-5
lines changed

6 files changed

+496
-5
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
1515
* Added support for multi-arity methods on `definterface` (#538)
1616
* Added support for Protocols (#460)
1717
* Added support for Volatiles (#460)
18+
* Add JSON encoder and decoder in `basilisp.json` namespace (#484)
1819

1920
### Fixed
2021
* Fixed a bug where the Basilisp AST nodes for return values of `deftype` members could be marked as _statements_ rather than _expressions_, resulting in an incorrect `nil` return (#523)

‎src/basilisp/core.lpy

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3019,6 +3019,17 @@
30193019
~@(rest method-calls)))
30203020
x))
30213021

3022+
(defmacro memfn
3023+
"Expands into a function that calls the method `name` on the first argument
3024+
of the resulting function. If `args` are provided, the resulting function will
3025+
have arguments of these names.
3026+
3027+
This is a convenient way of producing a first-class function for a Python
3028+
method."
3029+
[name & args]
3030+
`(fn [t# ~@args]
3031+
(. t# ~name ~@args)))
3032+
30223033
(defmacro new
30233034
"Create a new instance of class with args.
30243035

@@ -3942,6 +3953,8 @@
39423953
(:import opts)))]
39433954
`(do
39443955
(in-ns (quote ~name))
3956+
~(when doc
3957+
`(alter-meta! (the-ns (quote ~name)) assoc :doc ~doc))
39453958
(refer-basilisp ~@refer-filters)
39463959
~requires
39473960
~uses
@@ -4831,7 +4844,7 @@
48314844
(->> (group-by first methods)
48324845
(reduce (fn [m [method-name arities]]
48334846
(->> (map rest arities)
4834-
(apply list `fn method-name)
4847+
(apply list `fn)
48354848
(assoc m (keyword (name method-name)))))
48364849
{})))
48374850

‎src/basilisp/json.lpy

Lines changed: 249 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,249 @@
1+
(ns basilisp.json
2+
"JSON Encoder and Decoders
3+
4+
This namespace includes functions for performing basic JSON encoding from
5+
and decoding to Basilisp builtin data structures. It is built on top of Python's
6+
builtin `json` module. The builtin `json` module is not intended to be extended
7+
in the way that is done here. As such, it is not the fastest JSON decoder or
8+
encoder available, but it is builtin so it is readily available for quick
9+
encoding and decoding needs."
10+
(:refer-basilisp :exclude [read])
11+
(:import
12+
datetime
13+
decimal
14+
fractions
15+
json
16+
uuid))
17+
18+
;;;;;;;;;;;;;;
19+
;; Encoders ;;
20+
;;;;;;;;;;;;;;
21+
22+
(defprotocol JSONEncodeable
23+
(to-json-encodeable* [this opts]
24+
"Return an object which can be JSON encoded by Python's default JSONEncoder.
25+
26+
`opts` is a map with the following options:
27+
28+
`:key-fn` - is a function which will be called for each key in a map;
29+
default is `name`"))
30+
31+
(extend-protocol JSONEncodeable
32+
python/object
33+
(to-json-encodeable* [this _]
34+
(throw
35+
(python/TypeError
36+
(str "Cannot JSON encode objects of type " (python/type this))))))
37+
38+
(defn ^:private encodeable-scalar
39+
[o _]
40+
o)
41+
42+
(defn ^:private stringify-scalar
43+
[o _]
44+
(python/str o))
45+
46+
(defn ^:private encodeable-date-type
47+
[o _]
48+
(.isoformat o))
49+
50+
(defn ^:private kw-or-sym-to-encodeable
51+
[o _]
52+
(if-let [ns-str (namespace o)]
53+
(str ns-str "/" (name o))
54+
(name o)))
55+
56+
(defn ^:private map-to-encodeable
57+
[o {:keys [key-fn] :as opts}]
58+
(->> o
59+
(map (fn [[k v]] [(key-fn k) v]))
60+
(python/dict)))
61+
62+
(defn ^:private seq-to-encodeable
63+
[o opts]
64+
(->> o
65+
(map #(to-json-encodeable* % opts))
66+
(python/list)))
67+
68+
(extend python/str JSONEncodeable {:to-json-encodeable* encodeable-scalar})
69+
(extend python/int JSONEncodeable {:to-json-encodeable* encodeable-scalar})
70+
(extend python/float JSONEncodeable {:to-json-encodeable* encodeable-scalar})
71+
(extend python/bool JSONEncodeable {:to-json-encodeable* encodeable-scalar})
72+
(extend nil JSONEncodeable {:to-json-encodeable* encodeable-scalar})
73+
74+
(extend basilisp.lang.keyword/Keyword JSONEncodeable {:to-json-encodeable* kw-or-sym-to-encodeable})
75+
(extend basilisp.lang.symbol/Symbol JSONEncodeable {:to-json-encodeable* kw-or-sym-to-encodeable})
76+
77+
(extend basilisp.lang.interfaces/IPersistentMap JSONEncodeable {:to-json-encodeable* map-to-encodeable})
78+
79+
(extend basilisp.lang.interfaces/IPersistentList JSONEncodeable {:to-json-encodeable* seq-to-encodeable})
80+
(extend basilisp.lang.interfaces/IPersistentSet JSONEncodeable {:to-json-encodeable* seq-to-encodeable})
81+
(extend basilisp.lang.interfaces/IPersistentVector JSONEncodeable {:to-json-encodeable* seq-to-encodeable})
82+
83+
;; Support extended reader types.
84+
(extend datetime/datetime JSONEncodeable {:to-json-encodeable* encodeable-date-type})
85+
(extend datetime/date JSONEncodeable {:to-json-encodeable* encodeable-date-type})
86+
(extend datetime/time JSONEncodeable {:to-json-encodeable* encodeable-date-type})
87+
(extend decimal/Decimal JSONEncodeable {:to-json-encodeable* stringify-scalar})
88+
(extend fractions/Fraction JSONEncodeable {:to-json-encodeable* stringify-scalar})
89+
(extend uuid/UUID JSONEncodeable {:to-json-encodeable* stringify-scalar})
90+
91+
;; Support Python types in case they are embedded in other Basilisp collections.
92+
(extend python/dict JSONEncodeable {:to-json-encodeable* (fn [d opts] (map-to-encodeable (.items d) opts))})
93+
(extend python/list JSONEncodeable {:to-json-encodeable* seq-to-encodeable})
94+
(extend python/tuple JSONEncodeable {:to-json-encodeable* seq-to-encodeable})
95+
(extend python/set JSONEncodeable {:to-json-encodeable* seq-to-encodeable})
96+
(extend python/frozenset JSONEncodeable {:to-json-encodeable* seq-to-encodeable})
97+
98+
(defn ^:private write-opts
99+
[{:keys [escape-non-ascii indent item-sep key-fn key-sep]}]
100+
{:escape-non-ascii (if (boolean? escape-non-ascii) escape-non-ascii true)
101+
:indent indent
102+
:key-fn (or key-fn name)
103+
:separator #py ((or item-sep ", ") (or key-sep ": "))})
104+
105+
(defn write
106+
"Serialize the object `o` as JSON to the writer object `writer` (which must be
107+
any file-like object supporting `.write()` method).
108+
109+
All data structures supported by the Basilisp reader are serialized to JSON
110+
by default. Maps are serialized as JSON Objects. Lists, sets, and vectors are
111+
serialized as JSON arrays. Keywords and symbols are serialized as strings with
112+
their namespace (if they have one). Python scalar types are serialized as their
113+
corresponding JSON types (string, integer, float, boolean, and `nil`). Instants
114+
(Python `datetime`s) and the related Python `date` and `time` types are
115+
serialized as ISO 8601 date strings. Decimals are serialized as stringified
116+
floats. Fractions are serialized as stringified ratios (numerator and
117+
denominator). UUIDs are serialized as their canonical hex string format.
118+
119+
Support for other data structures can be added by extending the `JSONEncodeable`
120+
Protocol. That protocol includes one method which must return a Python data type
121+
which can be understood by Python's builtin `json` module.
122+
123+
The encoder supports a few options which may be specified as key/value pairs:
124+
125+
`:key-fn` - is a function which will be called for each key in a map;
126+
default is `name`
127+
`:escape-non-ascii` - if `true`, escape non-ASCII characters in the output;
128+
default is `true`
129+
`:indent` - if `nil`, use a compact representation; if a positive
130+
integer, each indent level will be that many spaces; if
131+
zero, a negative integer, or the empty string, newlines
132+
will be inserted without indenting; if a string, that
133+
string value will be used as the indent
134+
`:item-sep` - a string separator between object and array items;
135+
default is ', '
136+
`:key-sep` - a string separator between object key/value pairs;
137+
default is ': '"
138+
[o writer & {:as opts}]
139+
(let [{:keys [escape-non-ascii indent separator] :as opts} (write-opts opts)]
140+
(json/dump o writer **
141+
:default #(to-json-encodeable* % opts)
142+
:ensure-ascii escape-non-ascii
143+
:indent indent
144+
:separators separator)))
145+
146+
(defn write-str
147+
"Serialize the object `o` as JSON and return the serialized object as a string.
148+
149+
All data structures supported by the Basilisp reader are serialized to JSON
150+
by default. Maps are serialized as JSON Objects. Lists, sets, and vectors are
151+
serialized as JSON arrays. Keywords and symbols are serialized as strings with
152+
their namespace (if they have one). Python scalar types are serialized as their
153+
corresponding JSON types (string, integer, float, boolean, and `nil`). Instants
154+
(Python `datetime`s) and the related Python `date` and `time` types are
155+
serialized as ISO 8601 date strings. Decimals are serialized as stringified
156+
floats. Fractions are serialized as stringified ratios (numerator and
157+
denominator). UUIDs are serialized as their canonical hex string format.
158+
159+
Support for other data structures can be added by extending the `JSONEncodeable`
160+
Protocol. That protocol includes one method which must return a Python data type
161+
which can be understood by Python's builtin `json` module.
162+
163+
The options for `write-str` are the same as for those of `write`."
164+
[o & {:as opts}]
165+
(let [{:keys [escape-non-ascii indent separator] :as opts} (write-opts opts)]
166+
(json/dumps o **
167+
:default #(to-json-encodeable* % opts)
168+
:ensure-ascii escape-non-ascii
169+
:indent indent
170+
:separators separator)))
171+
172+
;;;;;;;;;;;;;;
173+
;; Decoders ;;
174+
;;;;;;;;;;;;;;
175+
176+
(defprotocol JSONDecodeable
177+
(from-decoded-json* [this opts]
178+
"Return a Basilisp object in place of a Python object returned by Python's
179+
default JSONDecoder.
180+
181+
`opts` is a map with the following options:
182+
183+
`:key-fn` - is a function which will be called for each key in a map;
184+
default is `identity`"))
185+
186+
(extend-protocol JSONDecodeable
187+
python/dict
188+
(from-decoded-json* [this {:keys [key-fn] :as opts}]
189+
(->> (.items this)
190+
(mapcat (fn [[k v]] [(key-fn k) (from-decoded-json* v opts)]))
191+
(apply hash-map)))
192+
193+
python/list
194+
(from-decoded-json* [this opts]
195+
(->> this (map #(from-decoded-json* % opts)) (vec))))
196+
197+
(defn ^:private decode-scalar
198+
[o _]
199+
o)
200+
201+
(extend python/int JSONDecodeable {:from-decoded-json* decode-scalar})
202+
(extend python/float JSONDecodeable {:from-decoded-json* decode-scalar})
203+
(extend python/str JSONDecodeable {:from-decoded-json* decode-scalar})
204+
(extend python/bool JSONDecodeable {:from-decoded-json* decode-scalar})
205+
(extend nil JSONDecodeable {:from-decoded-json* decode-scalar})
206+
207+
(defn ^:private read-opts
208+
[{:keys [key-fn strict?]}]
209+
{:key-fn (or key-fn identity)
210+
:strict (if (boolean? strict?) strict? true)})
211+
212+
;; Python's builtin `json.load` currently only includes an Object hook; it has
213+
;; no hook for Array types. Due to this limitation, we have to iteratively
214+
;; transform the entire parsed object into Basilisp data structures rather than
215+
;; building the final object iteratively. There is an open bug report with
216+
;; Python, but it has gotten no traction: https://bugs.python.org/issue36738
217+
218+
(defn read
219+
"Decode the JSON-encoded stream from `reader` (which can be any Python file-like
220+
object) into Basilisp data structures.
221+
222+
JSON Objects will be decoded as Basilisp maps. JSON Arrays will be decoded as
223+
as Basilisp vectors. All other JSON data types will be decoded as the
224+
corresponding Python types (strings, booleans, integers, floats, and `nil`).
225+
226+
The decoder supports a few options which may be specified as key/value pairs:
227+
228+
`:key-fn` - is a function which will be called for each key in a map;
229+
default is `identity`
230+
`:strict?` - boolean value; if `true`, control characters (characters in
231+
ASCII 0-31 range) will be prohibited inside JSON strings;
232+
default is `true`"
233+
[reader & {:as opts}]
234+
(let [{:keys [strict?] :as opts} (read-opts opts)]
235+
(-> (json/load reader ** :strict strict?)
236+
(from-decoded-json* opts))))
237+
238+
(defn read-str
239+
"Decode the JSON-encoded string `s` into Basilisp data structures.
240+
241+
JSON Objects will be decoded as Basilisp maps. JSON Arrays will be decoded as
242+
as Basilisp vectors. All other JSON data types will be decoded as the
243+
corresponding Python types (strings, booleans, integers, floats, and `nil`).
244+
245+
The options for `read-str` are the same as for those of `read`."
246+
[s & {:as opts}]
247+
(let [{:keys [strict?] :as opts} (read-opts opts)]
248+
(-> (json/loads s ** :strict strict?)
249+
(from-decoded-json* opts))))

‎src/basilisp/repl.lpy

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,20 @@
1717
(defn pydoc
1818
"Print the Python docstring for a function."
1919
[o]
20-
(print (inspect/getdoc o)))
20+
(println (inspect/getdoc o)))
2121

2222
(defn print-doc
2323
"Print the docstring from an interned var."
2424
[v]
2525
(let [var-meta (meta v)]
26-
(if var-meta
27-
(print (:doc var-meta))
28-
nil)))
26+
(println "------------------------")
27+
(println (cond->> (name v)
28+
(namespace v) (str (namespace v) "/")))
29+
(when var-meta
30+
(when-let [arglists (:arglists var-meta)]
31+
(println arglists))
32+
(when-let [docstring (:doc var-meta)]
33+
(println " " docstring)))))
2934

3035
(defmacro doc
3136
"Print the docstring from an interned Var if found."

‎tests/basilisp/prompt_test.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ def patch_completions(self, completions: Iterable[str]):
6868
"mapv",
6969
"max",
7070
"max-key",
71+
"memfn",
7172
"merge",
7273
"meta",
7374
"methods",

0 commit comments

Comments
 (0)