Skip to content

Commit f1458db

Browse files
sparkprimesparkprime
committed
Fix Off-by-one error in Python import and make the API compatible with 0.18.0.
1 parent a76a37d commit f1458db

File tree

5 files changed

+186
-18
lines changed

5 files changed

+186
-18
lines changed

‎python/_jsonnet.c

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -248,24 +248,33 @@ static int cpython_import_callback(void *ctx_, const char *base, const char *rel
248248
PyObject *file_name = PyTuple_GetItem(result, 0);
249249
PyObject *file_content = PyTuple_GetItem(result, 1);
250250
#if PY_MAJOR_VERSION >= 3
251-
if (!PyUnicode_Check(file_name) || !PyBytes_Check(file_content)) {
251+
if (!PyUnicode_Check(file_name) || !(PyUnicode_Check(file_content) || PyBytes_Check(file_content))) {
252252
#else
253-
if (!PyString_Check(file_name) || !PyString_Check(file_content)) {
253+
if (!PyString_Check(file_name) || !(PyString_Check(file_content) || PyBytes_Check(file_content))) {
254254
#endif
255-
*buf = jsonnet_str_nonull(ctx->vm, "import_callback did not return (string, bytes)", buflen);
255+
*buf = jsonnet_str_nonull(ctx->vm, "import_callback did not return (string, bytes) or (string, string)", buflen);
256256
success = 0;
257257
} else {
258258
const char *content_buf;
259-
const ssize_t content_len;
259+
ssize_t content_len;
260260
#if PY_MAJOR_VERSION >= 3
261261
const char *found_here_cstr = PyUnicode_AsUTF8(file_name);
262-
PyBytes_AsStringAndSize(file_content, &content_buf, &content_len);
263262
#else
264263
const char *found_here_cstr = PyString_AsString(file_name);
265-
PyString_AsStringAndSize(file_content, &content_buf, &content_len);
266264
#endif
265+
if (PyBytes_Check(file_content)) {
266+
PyBytes_AsStringAndSize(file_content, &content_buf, &content_len);
267+
} else {
268+
#if PY_MAJOR_VERSION >= 3
269+
content_buf = PyUnicode_AsUTF8(file_content);
270+
content_len = strlen(content_buf);
271+
#else
272+
content_buf = PyString_AsString(file_content);
273+
content_len = strlen(content_buf);
274+
#endif
275+
}
267276
*found_here = jsonnet_str(ctx->vm, found_here_cstr);
268-
*buflen = content_len - 1; // Python always adds a trailing null
277+
*buflen = content_len;
269278
*buf = jsonnet_realloc(ctx->vm, NULL, *buflen);
270279
memcpy(*buf, content_buf, *buflen);
271280
success = 1;

‎python/_jsonnet_test.py

Lines changed: 167 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,14 @@
1313
# limitations under the License.
1414

1515
import os
16+
import sys
1617
import unittest
1718

1819
import _jsonnet
1920

2021

2122
# Returns content if worked, None if file not found, or throws an exception
22-
def try_path(dir, rel):
23+
def try_path(dir, rel, do_encode):
2324
if not rel:
2425
raise RuntimeError('Got invalid filename (empty string).')
2526
if rel[0] == '/':
@@ -32,15 +33,31 @@ def try_path(dir, rel):
3233
if not os.path.isfile(full_path):
3334
return full_path, None
3435
with open(full_path) as f:
35-
return full_path, f.read().encode()
36+
if do_encode:
37+
return full_path, f.read().encode()
38+
else:
39+
return full_path, f.read()
3640

3741

38-
def import_callback(dir, rel):
39-
full_path, content = try_path(dir, rel)
42+
def import_callback_encode(dir, rel):
43+
full_path, content = try_path(dir, rel, do_encode=True)
4044
if content:
4145
return full_path, content
4246
raise RuntimeError('File not found')
4347

48+
def import_callback_no_encode(dir, rel):
49+
full_path, content = try_path(dir, rel, do_encode=False)
50+
if content:
51+
return full_path, content
52+
raise RuntimeError('File not found')
53+
54+
def import_callback_empty_file_encode(dir, rel):
55+
return dir, b''
56+
57+
58+
def import_callback_empty_file_no_encode(dir, rel):
59+
return dir, ''
60+
4461

4562
# Test native extensions
4663
def concat(a, b):
@@ -75,39 +92,178 @@ def setUp(self):
7592
with open(self.input_filename, "r") as infile:
7693
self.input_snippet = infile.read()
7794

78-
def test_evaluate_file(self):
95+
def test_evaluate_file_encode(self):
96+
json_str = _jsonnet.evaluate_file(
97+
self.input_filename,
98+
import_callback=import_callback_encode,
99+
native_callbacks=native_callbacks,
100+
)
101+
self.assertEqual(json_str, "true\n")
102+
103+
def test_evaluate_file_no_encode(self):
79104
json_str = _jsonnet.evaluate_file(
80105
self.input_filename,
81-
import_callback=import_callback,
106+
import_callback=import_callback_no_encode,
107+
native_callbacks=native_callbacks,
108+
)
109+
self.assertEqual(json_str, "true\n")
110+
111+
def test_evaluate_snippet_encode(self):
112+
json_str = _jsonnet.evaluate_snippet(
113+
self.test_filename,
114+
self.input_snippet,
115+
import_callback=import_callback_encode,
82116
native_callbacks=native_callbacks,
83117
)
84118
self.assertEqual(json_str, "true\n")
85119

86-
def test_evaluate_snippet(self):
120+
def test_evaluate_snippet_no_encode(self):
87121
json_str = _jsonnet.evaluate_snippet(
88122
self.test_filename,
89123
self.input_snippet,
90-
import_callback=import_callback,
124+
import_callback=import_callback_no_encode,
91125
native_callbacks=native_callbacks,
92126
)
93127
self.assertEqual(json_str, "true\n")
94128

95-
def test_import(self):
129+
def test_evaluate_snippet_encode(self):
130+
json_str = _jsonnet.evaluate_snippet(
131+
self.test_filename,
132+
self.input_snippet,
133+
import_callback=import_callback_encode,
134+
native_callbacks=native_callbacks,
135+
)
136+
self.assertEqual(json_str, "true\n")
137+
138+
def test_evaluate_snippet_no_encode(self):
139+
json_str = _jsonnet.evaluate_snippet(
140+
self.test_filename,
141+
self.input_snippet,
142+
import_callback=import_callback_no_encode,
143+
native_callbacks=native_callbacks,
144+
)
145+
self.assertEqual(json_str, "true\n")
146+
147+
def test_import_encode(self):
96148
json_str = _jsonnet.evaluate_snippet(
97149
self.test_filename,
98150
"import 'trivial.jsonnet'",
99-
import_callback=import_callback,
151+
import_callback=import_callback_encode,
152+
native_callbacks=native_callbacks,
153+
)
154+
self.assertEqual(json_str, "42\n")
155+
156+
def test_import_no_encode(self):
157+
json_str = _jsonnet.evaluate_snippet(
158+
self.test_filename,
159+
"import 'trivial.jsonnet'",
160+
import_callback=import_callback_no_encode,
161+
native_callbacks=native_callbacks,
162+
)
163+
self.assertEqual(json_str, "42\n")
164+
165+
def test_import_no_eol_encode(self):
166+
json_str = _jsonnet.evaluate_snippet(
167+
self.test_filename,
168+
"import 'trivial_no_eol.jsonnet'",
169+
import_callback=import_callback_encode,
100170
native_callbacks=native_callbacks,
101171
)
102172
self.assertEqual(json_str, "42\n")
103173

174+
def test_import_no_eol_no_encode(self):
175+
json_str = _jsonnet.evaluate_snippet(
176+
self.test_filename,
177+
"import 'trivial_no_eol.jsonnet'",
178+
import_callback=import_callback_no_encode,
179+
native_callbacks=native_callbacks,
180+
)
181+
self.assertEqual(json_str, "42\n")
182+
183+
def test_import_binary_encode(self):
184+
json_str = _jsonnet.evaluate_snippet(
185+
self.test_filename,
186+
"importbin 'binary123.bin'",
187+
import_callback=import_callback_encode,
188+
native_callbacks=native_callbacks,
189+
)
190+
self.assertEqual(json_str, "[\n 1,\n 2,\n 3\n]\n")
191+
192+
def test_import_binary_no_encode(self):
193+
json_str = _jsonnet.evaluate_snippet(
194+
self.test_filename,
195+
"importbin 'binary123.bin'",
196+
import_callback=import_callback_no_encode,
197+
native_callbacks=native_callbacks,
198+
)
199+
self.assertEqual(json_str, "[\n 1,\n 2,\n 3\n]\n")
200+
201+
def test_import_binary_sentinel_encode(self):
202+
json_str = _jsonnet.evaluate_snippet(
203+
self.test_filename,
204+
"importbin 'binary1230123.bin'",
205+
import_callback=import_callback_encode,
206+
native_callbacks=native_callbacks,
207+
)
208+
self.assertEqual(json_str, "[\n 1,\n 2,\n 3,\n 0,\n 1,\n 2,\n 3\n]\n")
209+
210+
def test_import_binary_sentinel_no_encode(self):
211+
json_str = _jsonnet.evaluate_snippet(
212+
self.test_filename,
213+
"importbin 'binary1230123.bin'",
214+
import_callback=import_callback_no_encode,
215+
native_callbacks=native_callbacks,
216+
)
217+
if sys.version_info.major < 3:
218+
# In Python2, the string can actually have a 0 in the middle
219+
self.assertEqual(json_str, "[\n 1,\n 2,\n 3,\n 0,\n 1,\n 2,\n 3\n]\n")
220+
else:
221+
# In Pyton3, the string is truncated
222+
self.assertEqual(json_str, "[\n 1,\n 2,\n 3\n]\n")
223+
224+
def test_import_str_empty_file_encode(self):
225+
json_str = _jsonnet.evaluate_snippet(
226+
self.test_filename,
227+
"importstr 'binary123.bin'",
228+
import_callback=import_callback_empty_file_encode,
229+
native_callbacks=native_callbacks,
230+
)
231+
self.assertEqual(json_str, "\"\"\n")
232+
233+
def test_import_str_empty_file_no_encode(self):
234+
json_str = _jsonnet.evaluate_snippet(
235+
self.test_filename,
236+
"importstr 'binary123.bin'",
237+
import_callback=import_callback_empty_file_no_encode,
238+
native_callbacks=native_callbacks,
239+
)
240+
self.assertEqual(json_str, "\"\"\n")
241+
242+
def test_import_binary_empty_file_encode(self):
243+
json_str = _jsonnet.evaluate_snippet(
244+
self.test_filename,
245+
"importbin 'binary123.bin'",
246+
import_callback=import_callback_empty_file_encode,
247+
native_callbacks=native_callbacks,
248+
)
249+
self.assertEqual(json_str, "[ ]\n")
250+
251+
def test_import_binary_empty_file_no_encode(self):
252+
json_str = _jsonnet.evaluate_snippet(
253+
self.test_filename,
254+
"importbin 'binary123.bin'",
255+
import_callback=import_callback_empty_file_no_encode,
256+
native_callbacks=native_callbacks,
257+
)
258+
self.assertEqual(json_str, "[ ]\n")
259+
104260
def test_double_import(self):
105261
json_str = _jsonnet.evaluate_snippet(
106262
self.test_filename,
107263
"local x = import 'trivial.jsonnet';\n" +
108264
"local y = import 'trivial.jsonnet';\n" +
109265
"x + y",
110-
import_callback=import_callback,
266+
import_callback=import_callback_encode,
111267
native_callbacks=native_callbacks,
112268
)
113269
self.assertEqual(json_str, "84\n")

‎python/testdata/binary123.bin

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+


‎python/testdata/binary1230123.bin

7 Bytes
Binary file not shown.

‎python/testdata/trivial_no_eol.jsonnet

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
// used for testing imports
2+
42

0 commit comments

Comments
 (0)