Skip to content

Commit e631566

Browse files
qunaibitqunaibit
committed
[GR-18002] Implement a TruffleLibrary for hashing storages
1 parent 7c7df1f commit e631566

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+2490
-3299
lines changed

‎graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/builtin/HashingTest.java

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
/*
2+
* Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
3+
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4+
*
5+
* The Universal Permissive License (UPL), Version 1.0
6+
*
7+
* Subject to the condition set forth below, permission is hereby granted to any
8+
* person obtaining a copy of this software, associated documentation and/or
9+
* data (collectively the "Software"), free of charge and under any and all
10+
* copyright rights in the Software, and any and all patent rights owned or
11+
* freely licensable by each licensor hereunder covering either (i) the
12+
* unmodified Software as contributed to or provided by such licensor, or (ii)
13+
* the Larger Works (as defined below), to deal in both
14+
*
15+
* (a) the Software, and
16+
*
17+
* (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
18+
* one is included with the Software each a "Larger Work" to which the Software
19+
* is contributed by such licensors),
20+
*
21+
* without restriction, including without limitation the rights to copy, create
22+
* derivative works of, display, perform, and distribute the Software and make,
23+
* use, sell, offer for sale, import, export, have made, and have sold the
24+
* Software and the Larger Work(s), and to sublicense the foregoing rights on
25+
* either these or other terms.
26+
*
27+
* This license is subject to the following condition:
28+
*
29+
* The above copyright notice and either this complete permission notice or at a
30+
* minimum a reference to the UPL must be included in all copies or substantial
31+
* portions of the Software.
32+
*
33+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
34+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
35+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
36+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
37+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
38+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
39+
* SOFTWARE.
40+
*/
41+
package com.oracle.graal.python.test.builtin;
42+
43+
import static com.oracle.graal.python.test.PythonTests.assertPrints;
44+
45+
import org.junit.Test;
46+
47+
public class HashingTest {
48+
49+
@Test
50+
public void mappingproxyTest() {
51+
String source = "_mappingproxy = type(type.__dict__)\n" +
52+
"d = {\"a\": 1, \"b\": 2, \"c\": 3}\n" +
53+
"mp = _mappingproxy(d)\n" +
54+
"assert len(mp) == 3\n" +
55+
"assert d.keys() == {'a', 'b', 'c'}\n" +
56+
"assert mp.keys() == d.keys()\n" +
57+
"assert len(mp.keys()) == 3, \"keys view has invalid length\"\n" +
58+
"assert set(mp.keys()) == {'a', 'b', 'c'}, \"keys view invalid\"\n" +
59+
"assert len(mp.values()) == 3, \"values view has invalid length\"\n" +
60+
"assert set(mp.values()) == {1, 2, 3}, \"values view invalid\"\n" +
61+
"assert len(mp.items()) == 3, \"items view has invalid length\"\n" +
62+
"assert set(mp.items()) == {('a', 1), ('b', 2), ('c', 3)}, \"items view invalid\"\n";
63+
assertPrints("", source);
64+
}
65+
66+
@Test
67+
public void customMappingObjectTest() {
68+
String source = "class CustomMappingObject:\n" +
69+
" def __init__(self, keys, values):\n" +
70+
" self._keys = keys\n" +
71+
" self._values = values\n" +
72+
" def __getitem__(self, k):\n" +
73+
" for i in range(len(self._keys)):\n" +
74+
" if k == self._keys[i]:\n" +
75+
" return self._values[i]\n" +
76+
" raise KeyError\n" +
77+
" def __setitem__(self, k, v):\n" +
78+
" for i in range(len(self._keys)):\n" +
79+
" if k == self._keys[i]:\n" +
80+
" self._values[i] = v\n" +
81+
" return v\n" +
82+
" raise KeyError\n" +
83+
" def keys(self):\n" +
84+
" return set(self._keys)\n" +
85+
" def values(self):\n" +
86+
" return self._values\n" +
87+
" def items(self):\n" +
88+
" return {(self._keys[i], self._values[i]) for i in range(len(self._keys))}\n" +
89+
" def __len__(self):\n" +
90+
" return len(self._keys)\n" +
91+
"_mappingproxy = type(type.__dict__)\n" +
92+
"mp_list = _mappingproxy(CustomMappingObject([\"a\", \"b\", \"c\"], [1, 2, 3]))\n" +
93+
"assert mp_list.keys() == {\"a\", \"b\", \"c\"}\n" +
94+
"";
95+
assertPrints("", source);
96+
}
97+
98+
@Test
99+
public void dictViewTest() {
100+
String source = "d = dict()\n" +
101+
"d['a'] = 1\n" +
102+
"d['b'] = 2\n" +
103+
"d['c'] = 3\n" +
104+
"assert len(d) == 3\n" +
105+
"assert len(d.keys()) == 3, \"keys view has invalid length\"\n" +
106+
"assert set(d.keys()) == {'a', 'b', 'c'}, \"keys view invalid\"\n" +
107+
"assert len(d.values()) == 3, \"values view has invalid length\"\n" +
108+
"assert set(d.values()) == {1, 2, 3}, \"values view invalid\"\n" +
109+
"assert len(d.items()) == 3, \"items view has invalid length\"\n" +
110+
"assert set(d.items()) == {('a', 1), ('b', 2), ('c', 3)}, \"items view invalid\"\n" +
111+
"";
112+
assertPrints("", source);
113+
}
114+
115+
@Test
116+
public void dictEqualTest1() {
117+
String source = "d = dict.fromkeys(['a', 'b', 'c'])\n" +
118+
"assert len(d) == 3\n" +
119+
"assert set(d.keys()) == {'a', 'b', 'c'}\n" +
120+
"assert set(d.values()) == {None}\n" +
121+
"d = dict.fromkeys(['a', 'b', 'c'], 1)\n" +
122+
"assert len(d) == 3\n" +
123+
"assert set(d.keys()) == {'a', 'b', 'c'}\n" +
124+
"assert set(d.values()) == {1}\n" +
125+
"d = dict.fromkeys(['a', 'b', 'c'], 1.0)\n" +
126+
"assert len(d) == 3\n" +
127+
"assert set(d.keys()) == {'a', 'b', 'c'}\n" +
128+
"assert set(d.values()) == {1.0}\n" +
129+
"d = dict.fromkeys(['a', 'b', 'c'], 'd')\n" +
130+
"assert len(d) == 3\n" +
131+
"assert set(d.keys()) == {'a', 'b', 'c'}\n" +
132+
"assert set(d.values()) == {'d'}\n" +
133+
"d = dict.fromkeys(['a', 'b', 'c'], None)\n" +
134+
"assert len(d) == 3\n" +
135+
"assert set(d.keys()) == {'a', 'b', 'c'}\n" +
136+
"assert set(d.values()) == {None}\n" +
137+
"o = object()\n" +
138+
"d = dict.fromkeys(['a', 'b', 'c'], o)\n" +
139+
"assert len(d) == 3\n" +
140+
"assert set(d.keys()) == {'a', 'b', 'c'}\n" +
141+
"assert set(d.values()) == {o}\n" +
142+
"";
143+
assertPrints("", source);
144+
}
145+
146+
@Test
147+
public void dictEqualTest2() {
148+
String source = "d = dict(a=1, b=2, c=3)\n" +
149+
"def assert_raises(err, fn, *args, **kwargs):\n" +
150+
" raised = False\n" +
151+
" try:\n" +
152+
" fn(*args, **kwargs)\n" +
153+
" except err:\n" +
154+
" raised = True\n" +
155+
" assert raised\n" +
156+
"assert len(d) == 3\n" +
157+
"assert set(d.keys()) == {'a', 'b', 'c'}\n" +
158+
"d = dict.fromkeys(['a', 'b', 'c'])\n" +
159+
"assert len(d) == 3\n" +
160+
"assert set(d.keys()) == {'a', 'b', 'c'}\n" +
161+
"assert list(d.values()) == [None, None, None]\n" +
162+
"d = dict.fromkeys(['a', 'b', 'c'], 1)\n" +
163+
"assert len(d) == 3\n" +
164+
"assert set(d.keys()) == {'a', 'b', 'c'}\n" +
165+
"assert list(d.values()) == [1, 1, 1]\n" +
166+
"assert_raises(TypeError, dict.fromkeys, 10)\n" +
167+
"";
168+
assertPrints("", source);
169+
}
170+
171+
@Test
172+
public void dictEqualTest3() {
173+
String source = "key_set = {'a', 'b', 'c', 'd'}\n" +
174+
"\n" +
175+
"class CustomMappingObject:\n" +
176+
" def __init__(self, keys):\n" +
177+
" self.__keys = keys\n" +
178+
"\n" +
179+
" def keys(self):\n" +
180+
" return self.__keys\n" +
181+
"\n" +
182+
" def __getitem__(self, key):\n" +
183+
" if key in self.__keys:\n" +
184+
" return ord(key)\n" +
185+
" raise KeyError(key)\n" +
186+
"\n" +
187+
" def __len__(self):\n" +
188+
" return len(self.keys)\n" +
189+
"\n" +
190+
"d = dict(CustomMappingObject(key_set))\n" +
191+
"assert len(d) == 4, \"invalid length, expected 4 but was %d\" % len(d)\n" +
192+
"assert set(d.keys()) == key_set, \"unexpected keys: %s\" % str(d.keys())\n" +
193+
"assert set(d.values()) == {97, 98, 99, 100}, \"unexpected values: %s\" % str(d.values())\n" +
194+
"";
195+
assertPrints("", source);
196+
}
197+
198+
}

‎graalpython/com.oracle.graal.python.test/src/tests/cpyext/__init__.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
1+
# Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
22
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
33
#
44
# The Universal Permissive License (UPL), Version 1.0
@@ -415,7 +415,7 @@ def create_module(self, name=None):
415415
fargs["resultvarlocations"] = ", ".join("&" + arg.rpartition(" ")[2] for arg in fargs["resultvars"])
416416
if "resulttype" not in fargs:
417417
fargs["resulttype"] = "void*"
418-
if len(fargs["resultvarlocations"]):
418+
if len(fargs["resultvarlocations"]) and not fargs["resultvarlocations"].startswith(","):
419419
fargs["resultvarlocations"] = ", " + fargs["resultvarlocations"]
420420
self._insert(fargs, "customcode", "")
421421
super(CPyExtFunctionOutVars, self).create_module(name)

‎graalpython/com.oracle.graal.python.test/src/tests/test_dict.py

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
1+
# Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
22
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
33
#
44
# The Universal Permissive License (UPL), Version 1.0
@@ -597,3 +597,34 @@ def __eq__(self, other):
597597
assert b in d
598598
assert count_hash == 4, count_hash
599599
assert count_eq == 1, count_eq
600+
601+
602+
def test_hash_and_eq_for_dynamic_object_storage():
603+
class MyObject:
604+
def __init__(self, string):
605+
self.string = string
606+
607+
def __eq__(self, other):
608+
return self.string == other
609+
610+
def __hash__(self):
611+
return hash(self.string)
612+
613+
d = {"1": 42}
614+
615+
d2 = MyObject("1").__dict__
616+
d2["1"] = 42
617+
618+
assert MyObject("1") in d
619+
assert d[MyObject("1")] == 42
620+
d[MyObject("1")] = 112
621+
assert d[MyObject("1")] == 112
622+
del d[MyObject("1")]
623+
assert "1" not in d
624+
625+
assert MyObject("1") in d2
626+
assert d2[MyObject("1")] == 42
627+
d2[MyObject("1")] = 112
628+
assert d2[MyObject("1")] == 112
629+
del d2[MyObject("1")]
630+
assert "1" not in d2

‎graalpython/com.oracle.graal.python.test/src/tests/test_set.py

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
1+
# Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
22
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
33
#
44
# The Universal Permissive License (UPL), Version 1.0
@@ -323,3 +323,10 @@ def test_hashable_frozenset():
323323
d = {key1: 42}
324324
assert hash(key1) == hash(key2)
325325
assert d[key2] == 42
326+
327+
328+
def test_equality():
329+
s1 = {1, 2, 3}
330+
s2 = {1, 3}
331+
assert not s1 == s2
332+
assert not s2 == s1

‎graalpython/com.oracle.graal.python.test/src/tests/test_type.py

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
1+
# Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
22
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
33
#
44
# The Universal Permissive License (UPL), Version 1.0
@@ -70,3 +70,14 @@ def test_base():
7070
#
7171
# C = type('C', (B, int), {'spam': lambda self: 'spam%s' % self})
7272
# assert C.__base__ == int
73+
74+
75+
def test_namespace_with_non_string_keys():
76+
class MyStr(str):
77+
pass
78+
79+
A = type('A', (), {
80+
MyStr("x"): 42
81+
})
82+
assert A.x == 42
83+
assert any(type(k) == MyStr for k in A.__dict__.keys())

‎graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_dict.txt

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
*DictTest.test_dictview_set_operations_on_keys
1414
*DictTest.test_empty_presized_dict_in_freelist
1515
*DictTest.test_eq
16-
*DictTest.test_equal_operator_modifying_operand
1716
*DictTest.test_errors_in_view_containment_check
1817
*DictTest.test_fromkeys_operator_modifying_dict_operand
1918
*DictTest.test_fromkeys_operator_modifying_set_operand

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinConstructors.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1157,7 +1157,7 @@ public PFrozenSet frozensetEmpty(LazyPythonClass cls, @SuppressWarnings("unused"
11571157
public PFrozenSet frozenset(VirtualFrame frame, LazyPythonClass cls, String arg) {
11581158
PFrozenSet frozenSet = factory().createFrozenSet(cls);
11591159
for (int i = 0; i < PString.length(arg); i++) {
1160-
getSetItemNode().execute(frame, frozenSet, PString.valueOf(PString.charAt(arg, i)), PNone.NO_VALUE);
1160+
getSetItemNode().execute(frame, frozenSet, PString.valueOf(PString.charAt(arg, i)), PNone.NONE);
11611161
}
11621162
return frozenSet;
11631163
}
@@ -1172,7 +1172,7 @@ public PFrozenSet frozensetIterable(VirtualFrame frame, LazyPythonClass cls, Obj
11721172
PFrozenSet frozenSet = factory().createFrozenSet(cls);
11731173
while (true) {
11741174
try {
1175-
getSetItemNode().execute(frame, frozenSet, next.execute(frame, iterator), PNone.NO_VALUE);
1175+
getSetItemNode().execute(frame, frozenSet, next.execute(frame, iterator), PNone.NONE);
11761176
} catch (PException e) {
11771177
e.expectStopIteration(errorProfile);
11781178
return frozenSet;

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/CodecsModuleBuiltins.java

Lines changed: 23 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -64,9 +64,12 @@
6464
import com.oracle.graal.python.builtins.objects.bytes.BytesNodes;
6565
import com.oracle.graal.python.builtins.objects.bytes.PBytes;
6666
import com.oracle.graal.python.builtins.objects.bytes.PIBytesLike;
67+
import com.oracle.graal.python.builtins.objects.common.HashingStorage;
68+
import com.oracle.graal.python.builtins.objects.common.HashingStorageLibrary;
6769
import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes;
6870
import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes.GetInternalByteArrayNode;
6971
import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodesFactory.GetInternalByteArrayNodeGen;
72+
import com.oracle.graal.python.builtins.objects.dict.PDict;
7073
import com.oracle.graal.python.builtins.objects.tuple.PTuple;
7174
import com.oracle.graal.python.nodes.expression.CoerceToBooleanNode;
7275
import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode;
@@ -86,6 +89,7 @@
8689
import com.oracle.truffle.api.dsl.Specialization;
8790
import com.oracle.truffle.api.dsl.TypeSystemReference;
8891
import com.oracle.truffle.api.frame.VirtualFrame;
92+
import com.oracle.truffle.api.library.CachedLibrary;
8993
import com.oracle.truffle.api.profiles.ValueProfile;
9094

9195
@CoreFunctions(defineModule = "_codecs")
@@ -670,24 +674,31 @@ Object lookup(String encoding) {
670674
abstract static class CharmapBuildNode extends PythonBuiltinNode {
671675
// This is replaced in the core _codecs.py with the full functionality
672676
@Specialization
673-
Object lookup(String chars) {
674-
Map<Integer, Integer> charmap = createMap(chars);
675-
return factory().createDict(charmap);
676-
}
677-
678-
@TruffleBoundary
679-
private static Map<Integer, Integer> createMap(String chars) {
680-
Map<Integer, Integer> charmap = new HashMap<>();
677+
Object lookup(String chars,
678+
@CachedLibrary(limit = "3") HashingStorageLibrary lib) {
679+
HashingStorage store = PDict.createNewStorage(false, chars.length());
680+
PDict dict = factory().createDict(store);
681681
int pos = 0;
682682
int num = 0;
683683

684684
while (pos < chars.length()) {
685-
int charid = Character.codePointAt(chars, pos);
686-
charmap.put(charid, num);
687-
pos += Character.charCount(charid);
685+
int charid = codePointAt(chars, pos);
686+
store = lib.setItem(store, charid, num);
687+
pos += charCount(charid);
688688
num++;
689689
}
690-
return charmap;
690+
dict.setDictStorage(store);
691+
return dict;
692+
}
693+
694+
@TruffleBoundary
695+
private static int charCount(int charid) {
696+
return Character.charCount(charid);
697+
}
698+
699+
@TruffleBoundary
700+
private static int codePointAt(String chars, int pos) {
701+
return Character.codePointAt(chars, pos);
691702
}
692703
}
693704
}

0 commit comments

Comments
 (0)