Skip to content

Commit 7453e06

Browse files
committed
Improve the formatting support
* rework the printf style formatting to support bytes/bytearray * add format method to complex numbers * unify code in String's repr and AsciiNode and use ICU4J to identify printable characters * make some more test_format tests pass
1 parent 0784cb2 commit 7453e06

25 files changed

+1606
-445
lines changed

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ GRAALPYTHON_UNIT_TESTS.dist
2121
mx.graalpython/eclipse-launches
2222
asv
2323
*.json
24+
!**/resources/*.json
2425
language
2526
*.bc
2627
*.iml
Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
# Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
2+
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
3+
#
4+
# The Universal Permissive License (UPL), Version 1.0
5+
#
6+
# Subject to the condition set forth below, permission is hereby granted to any
7+
# person obtaining a copy of this software, associated documentation and/or
8+
# data (collectively the "Software"), free of charge and under any and all
9+
# copyright rights in the Software, and any and all patent rights owned or
10+
# freely licensable by each licensor hereunder covering either (i) the
11+
# unmodified Software as contributed to or provided by such licensor, or (ii)
12+
# the Larger Works (as defined below), to deal in both
13+
#
14+
# (a) the Software, and
15+
#
16+
# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
17+
# one is included with the Software each a "Larger Work" to which the Software
18+
# is contributed by such licensors),
19+
#
20+
# without restriction, including without limitation the rights to copy, create
21+
# derivative works of, display, perform, and distribute the Software and make,
22+
# use, sell, offer for sale, import, export, have made, and have sold the
23+
# Software and the Larger Work(s), and to sublicense the foregoing rights on
24+
# either these or other terms.
25+
#
26+
# This license is subject to the following condition:
27+
#
28+
# The above copyright notice and either this complete permission notice or at a
29+
# minimum a reference to the UPL must be included in all copies or substantial
30+
# portions of the Software.
31+
#
32+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
33+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
34+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
35+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
36+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
37+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
38+
# SOFTWARE.
39+
40+
import unittest
41+
42+
43+
class Polymorph:
44+
def __index__(self):
45+
return 42
46+
def __int__(self):
47+
return 1
48+
def __float__(self):
49+
return 3.14
50+
def __str__(self):
51+
return "hello"
52+
def __bytes__(self):
53+
return b"bytes"
54+
55+
56+
def test_formatting():
57+
# tests some corner-cases that the standard tests do not cover
58+
assert format(-12e8, "0=30,.4f") == '-0,000,000,001,200,000,000.0000'
59+
assert b"%(mykey)d" % {b'mykey': 42} == b"42"
60+
assert b"%c" % b'q' == b"q"
61+
assert "%-5d" % 42 == "42 "
62+
assert "%.*f" % (-2, 2.5) == "2"
63+
assert "%.*f" % (True, 2.51) == "2.5"
64+
assert "%ld" % 42 == "42"
65+
66+
assert "%c" % Polymorph() == "*"
67+
assert "%d" % Polymorph() == "1"
68+
assert "%x" % Polymorph() == "2a"
69+
assert "%s" % Polymorph() == "hello"
70+
assert "%.2f" % Polymorph() == "3.14"
71+
assert b"%c" % Polymorph() == b"*"
72+
assert b"%s" % Polymorph() == b"bytes"
73+
74+
assert type(bytearray("hello %d", "ascii") % 42) == bytearray
75+
assert type(b"hello %d" % 42) == bytes
76+
77+
78+
def test_complex_formatting():
79+
assert format(3+2j, ">20,.4f") == " 3.0000+2.0000j"
80+
assert format(3+2j, "+.2f") == "+3.00+2.00j"
81+
assert format(-3+2j, "+.2f") == "-3.00+2.00j"
82+
assert format(3+2j, "-.3f") == "3.000+2.000j"
83+
assert format(3-2j, "-.3f") == "3.000-2.000j"
84+
assert format(-3-2j, "-.3f") == "-3.000-2.000j"
85+
assert format(3+2j, " .1f") == " 3.0+2.0j"
86+
assert format(-3+2j, " .1f") == "-3.0+2.0j"
87+
assert format(complex(3), ".1g") == "3+0j"
88+
assert format(3j, ".1g") == "0+3j"
89+
assert format(-3j, ".1g") == "-0-3j"
90+
assert format(3j, "") == "3j"
91+
assert format(1+0j, "") == "(1+0j)"
92+
assert format(1+2j, "") == "(1+2j)"
93+
assert format(complex(1, float("NaN")), "") == "(1+nanj)"
94+
assert format(complex(1, float("Inf")), "") == "(1+infj)"
95+
96+
97+
class AnyRepr:
98+
def __init__(self, val):
99+
self.val = val
100+
def __repr__(self):
101+
return self.val
102+
103+
104+
def test_non_ascii_repr():
105+
assert "%a" % AnyRepr("\t") == "\t"
106+
assert "%a" % AnyRepr("\\") == "\\"
107+
assert "%a" % AnyRepr("\\") == "\\"
108+
assert "%a" % AnyRepr("\u0378") == "\\u0378"
109+
assert "%r" % AnyRepr("\u0378") == "\u0378"
110+
assert "%a" % AnyRepr("\u0374") == "\\u0374"
111+
assert "%r" % AnyRepr("\u0374") == "\u0374"
112+
113+
assert b"%a" % AnyRepr("\t") == b"\t"
114+
assert b"%a" % AnyRepr("\\") == b"\\"
115+
assert b"%a" % AnyRepr("\\") == b"\\"
116+
assert b"%a" % AnyRepr("\u0378") == b"\\u0378"
117+
assert b"%r" % AnyRepr("\u0378") == b"\\u0378"
118+
assert b"%a" % AnyRepr("\u0374") == b"\\u0374"
119+
assert b"%r" % AnyRepr("\u0374") == b"\\u0374"
120+
121+
122+
class FormattingErrorsTest(unittest.TestCase):
123+
def test_formatting_errors(self):
124+
self.assertRaises(TypeError, lambda: format(-12e8, b"0=30,.4f"))
125+
self.assertRaises(TypeError, lambda: format(42, b"0=30,.4f"))
126+
self.assertRaises(TypeError, lambda: format("str", b"0=30,.4f"))
127+
self.assertRaises(TypeError, lambda: format(3+1j, b"0=30,.4f"))
128+
self.assertRaises(TypeError, lambda: b"hello" % b"world")
129+
self.assertRaises(TypeError, lambda: b"%f" % "str")
130+
self.assertRaises(TypeError, lambda: b"%c" % "str")
131+
132+
self.assertRaises(KeyError, lambda: b"%(mykey)d" % {"mykey": 42})
133+
self.assertRaises(KeyError, lambda: "%(mykey)d" % {b"mykey": 42})
134+
self.assertRaises(OverflowError, lambda: b"%c" % 260)
135+
136+
self.assertRaises(ValueError, lambda: format(3+2j, "f=30,.4f"))
137+
self.assertRaises(ValueError, lambda: format(3+2j, "0=30,.4f"))
Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1-
*graalpython.lib-python.3.test.test_format.FormatTest.test_format_class
1+
*graalpython.lib-python.3.test.test_format.FormatTest.test_common_format
2+
*graalpython.lib-python.3.test.test_format.FormatTest.test_str_format
3+
*graalpython.lib-python.3.test.test_format.FormatTest.test_bytes_and_bytearray_format
4+
*graalpython.lib-python.3.test.test_format.FormatTest.test_nul
25
*graalpython.lib-python.3.test.test_format.FormatTest.test_optimisations
6+
*graalpython.lib-python.3.test.test_format.FormatTest.test_precision
37
*graalpython.lib-python.3.test.test_format.FormatTest.test_precision_c_limits
8+
*graalpython.lib-python.3.test.test_format.FormatTest.test_format_class

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SysModuleBuiltins.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@
8383
import com.oracle.graal.python.runtime.PythonCore;
8484
import com.oracle.graal.python.runtime.PythonOptions;
8585
import com.oracle.graal.python.runtime.exception.PException;
86+
import com.oracle.graal.python.runtime.formatting.IntegerFormatter;
8687
import com.oracle.graal.python.runtime.object.PythonObjectFactory;
8788
import com.oracle.graal.python.util.CharsetMapping;
8889
import com.oracle.graal.python.util.OverflowException;
@@ -153,7 +154,7 @@ public void initialize(PythonCore core) {
153154
2, // FLT_RADIX
154155
1 // FLT_ROUNDS
155156
}));
156-
builtinConstants.put("maxunicode", Character.MAX_CODE_POINT);
157+
builtinConstants.put("maxunicode", IntegerFormatter.LIMIT_UNICODE.intValue() - 1);
157158

158159
String os = getPythonOSName();
159160
builtinConstants.put("platform", os);

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/bytes/BytesBuiltins.java

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import static com.oracle.graal.python.nodes.SpecialMethodNames.__LEN__;
3939
import static com.oracle.graal.python.nodes.SpecialMethodNames.__LE__;
4040
import static com.oracle.graal.python.nodes.SpecialMethodNames.__LT__;
41+
import static com.oracle.graal.python.nodes.SpecialMethodNames.__MOD__;
4142
import static com.oracle.graal.python.nodes.SpecialMethodNames.__MUL__;
4243
import static com.oracle.graal.python.nodes.SpecialMethodNames.__NE__;
4344
import static com.oracle.graal.python.nodes.SpecialMethodNames.__REPR__;
@@ -52,6 +53,7 @@
5253
import java.util.Arrays;
5354
import java.util.List;
5455

56+
import com.oracle.graal.python.PythonLanguage;
5557
import com.oracle.graal.python.builtins.Builtin;
5658
import com.oracle.graal.python.builtins.CoreFunctions;
5759
import com.oracle.graal.python.builtins.PythonBuiltinClassType;
@@ -74,13 +76,15 @@
7476
import com.oracle.graal.python.builtins.objects.memoryview.PMemoryView;
7577
import com.oracle.graal.python.builtins.objects.object.PythonObjectLibrary;
7678
import com.oracle.graal.python.builtins.objects.tuple.PTuple;
79+
import com.oracle.graal.python.builtins.objects.tuple.TupleBuiltins;
7780
import com.oracle.graal.python.builtins.objects.type.TypeNodes;
7881
import com.oracle.graal.python.nodes.ErrorMessages;
7982
import com.oracle.graal.python.nodes.PGuards;
8083
import com.oracle.graal.python.nodes.PRaiseNode;
8184
import com.oracle.graal.python.nodes.SpecialMethodNames;
8285
import com.oracle.graal.python.nodes.argument.ReadArgumentNode;
8386
import com.oracle.graal.python.nodes.builtins.ListNodes.AppendNode;
87+
import com.oracle.graal.python.nodes.call.special.LookupAndCallBinaryNode;
8488
import com.oracle.graal.python.nodes.call.special.LookupAndCallUnaryNode;
8589
import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
8690
import com.oracle.graal.python.nodes.function.PythonBuiltinNode;
@@ -92,7 +96,10 @@
9296
import com.oracle.graal.python.nodes.truffle.PythonArithmeticTypes;
9397
import com.oracle.graal.python.nodes.util.CastToByteNode;
9498
import com.oracle.graal.python.nodes.util.CastToJavaIntExactNode;
99+
import com.oracle.graal.python.runtime.ExecutionContext.IndirectCallContext;
100+
import com.oracle.graal.python.runtime.PythonContext;
95101
import com.oracle.graal.python.runtime.exception.PythonErrorType;
102+
import com.oracle.graal.python.runtime.formatting.BytesFormatProcessor;
96103
import com.oracle.graal.python.runtime.sequence.storage.ByteSequenceStorage;
97104
import com.oracle.graal.python.runtime.sequence.storage.IntSequenceStorage;
98105
import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage;
@@ -103,6 +110,7 @@
103110
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
104111
import com.oracle.truffle.api.dsl.Cached;
105112
import com.oracle.truffle.api.dsl.Cached.Shared;
113+
import com.oracle.truffle.api.dsl.CachedContext;
106114
import com.oracle.truffle.api.dsl.Fallback;
107115
import com.oracle.truffle.api.dsl.GenerateNodeFactory;
108116
import com.oracle.truffle.api.dsl.GenerateUncached;
@@ -439,6 +447,46 @@ public Object mul(Object self, Object other) {
439447
}
440448
}
441449

450+
@Builtin(name = __MOD__, minNumOfPositionalArgs = 2)
451+
@GenerateNodeFactory
452+
abstract static class ModNode extends PythonBinaryBuiltinNode {
453+
454+
@Specialization(limit = "3")
455+
Object doBytes(VirtualFrame frame, PBytes self, Object right,
456+
@CachedLibrary("self") PythonObjectLibrary selfLib,
457+
@Cached("create(__GETITEM__)") LookupAndCallBinaryNode getItemNode,
458+
@Cached TupleBuiltins.GetItemNode getTupleItemNode,
459+
@CachedContext(PythonLanguage.class) PythonContext context) {
460+
byte[] data = format(frame, self, right, selfLib, getItemNode, getTupleItemNode, context);
461+
return factory().createBytes(data);
462+
}
463+
464+
@Specialization(limit = "3")
465+
Object doByteArray(VirtualFrame frame, PByteArray self, Object right,
466+
@CachedLibrary("self") PythonObjectLibrary selfLib,
467+
@Cached("create(__GETITEM__)") LookupAndCallBinaryNode getItemNode,
468+
@Cached TupleBuiltins.GetItemNode getTupleItemNode,
469+
@CachedContext(PythonLanguage.class) PythonContext context) {
470+
byte[] data = format(frame, self, right, selfLib, getItemNode, getTupleItemNode, context);
471+
return factory().createByteArray(data);
472+
}
473+
474+
private byte[] format(VirtualFrame frame, Object self, Object right, PythonObjectLibrary selfLib, LookupAndCallBinaryNode getItemNode, TupleBuiltins.GetItemNode getTupleItemNode,
475+
PythonContext context) {
476+
assert self instanceof PBytes || self instanceof PByteArray;
477+
Object state = IndirectCallContext.enter(frame, context, this);
478+
try {
479+
BytesFormatProcessor formatter = new BytesFormatProcessor(context.getCore(), getItemNode, getTupleItemNode, selfLib.getBufferBytes(self));
480+
return formatter.format(right);
481+
} catch (UnsupportedMessageException e) {
482+
CompilerDirectives.transferToInterpreter();
483+
throw new IllegalStateException();
484+
} finally {
485+
IndirectCallContext.exit(frame, context, state);
486+
}
487+
}
488+
}
489+
442490
@Builtin(name = __REPR__, minNumOfPositionalArgs = 1)
443491
@GenerateNodeFactory
444492
public abstract static class ReprNode extends PythonUnaryBuiltinNode {
@@ -1929,10 +1977,12 @@ public abstract static class BytesLikeNoGeneralizationNode extends SequenceStora
19291977

19301978
public static final GenNodeSupplier SUPPLIER = new GenNodeSupplier() {
19311979

1980+
@Override
19321981
public GeneralizationNode create() {
19331982
return BytesLikeNoGeneralizationNodeGen.create();
19341983
}
19351984

1985+
@Override
19361986
public GeneralizationNode getUncached() {
19371987
return BytesLikeNoGeneralizationNodeGen.getUncached();
19381988
}

0 commit comments

Comments
 (0)