[GR-35417] Intrinsify _csv module.

PullRequest: graalpython/2034

Author: phoeinx

CHANGELOG.md

@@ -6,6 +6,7 @@ language runtime. The main focus is on user-observable behavior of the engine.
 ## Version 22.0.0
 * Added support for `pyexpat` module.
 * Added partial support for `PYTHONHASHSEED` environment variable (also available via `HashSeed` context option), currently only affecting hashing in `pyexpat` module.
+* Implement `_csv` module.
 * Improved compatibility with PyPI packages `wheel` and `click`
 
 ## Version 21.3.0

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/nodes/util/PyObjectLookupAttrTests.java

@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package com.oracle.graal.python.nodes.util;
+
+import com.oracle.graal.python.builtins.PythonBuiltinClassType;
+import com.oracle.graal.python.builtins.objects.PNone;
+import com.oracle.graal.python.lib.PyObjectLookupAttr;
+import com.oracle.truffle.api.frame.VirtualFrame;
+import com.oracle.truffle.api.nodes.Node;
+import com.oracle.truffle.api.nodes.RootNode;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.oracle.graal.python.test.PythonTests;
+
+public class PyObjectLookupAttrTests {
+
+    @Before
+    public void setUp() {
+        PythonTests.enterContext();
+    }
+
+    @After
+    public void tearDown() {
+        PythonTests.closeContext();
+    }
+
+    // Regression test to ensure that super class attributes are evaluated for BuiltinClassType
+    // attribute lookup.
+    // Booleans super class PInt defines "real" which we expect to find in an attribute lookup.
+    @Test
+    public void lookupThroughMRO() {
+        Object v = new RootNode(null) {
+            @Node.Child private PyObjectLookupAttr lookupNode = PyObjectLookupAttr.create();
+
+            @Override
+            public Object execute(VirtualFrame frame) {
+                return lookupNode.execute(frame, PythonBuiltinClassType.Boolean, "real");
+            }
+        }.getCallTarget().call();
+        Assert.assertNotSame(PNone.NO_VALUE, v);
+    }
+}

graalpython/com.oracle.graal.python.test/src/tests/test_csv.py

@@ -0,0 +1,79 @@
+# Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# The Universal Permissive License (UPL), Version 1.0
+#
+# Subject to the condition set forth below, permission is hereby granted to any
+# person obtaining a copy of this software, associated documentation and/or
+# data (collectively the "Software"), free of charge and under any and all
+# copyright rights in the Software, and any and all patent rights owned or
+# freely licensable by each licensor hereunder covering either (i) the
+# unmodified Software as contributed to or provided by such licensor, or (ii)
+# the Larger Works (as defined below), to deal in both
+#
+# (a) the Software, and
+#
+# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+# one is included with the Software each a "Larger Work" to which the Software
+# is contributed by such licensors),
+#
+# without restriction, including without limitation the rights to copy, create
+# derivative works of, display, perform, and distribute the Software and make,
+# use, sell, offer for sale, import, export, have made, and have sold the
+# Software and the Larger Work(s), and to sublicense the foregoing rights on
+# either these or other terms.
+#
+# This license is subject to the following condition:
+#
+# The above copyright notice and either this complete permission notice or at a
+# minimum a reference to the UPL must be included in all copies or substantial
+# portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import unittest
+import csv
+from tempfile import TemporaryFile
+
+class CsvTest(unittest.TestCase):
+    def test_read_utf_32_delimiter(self):
+        test_data = ['a\U0001F642b']
+        reader = csv.reader(test_data, delimiter="\U0001F642")
+        self.assertEqual(list(reader), [['a', 'b']])
+
+    def test_read_utf_32_field(self):
+        test_data = ['a,\U0001F642,b']
+        reader = csv.reader(test_data)
+        self.assertEqual(list(reader), [['a', '\U0001F642', 'b']])
+
+    def test_write_utf32_field(self):
+        with TemporaryFile("w+", newline='') as fileobj:
+            writer = csv.writer(fileobj)
+            writer.writerow(['a', '\U0001F642','b'])
+
+            expected = 'a,\U0001F642,b'
+            fileobj.seek(0)
+
+            self.assertEqual(fileobj.read(),
+                             expected + writer.dialect.lineterminator)
+
+    def test_write_utf32_field_with_utf32_delimiter(self):
+        with TemporaryFile("w+", newline='') as fileobj:
+            writer = csv.writer(fileobj, delimiter = '\U0001F642')
+            writer.writerow(['a', '\U0001F642','b'])
+
+            expected = 'a\U0001F642"\U0001F642"\U0001F642b'
+            fileobj.seek(0)
+
+            self.assertEqual(fileobj.read(),
+                             expected + writer.dialect.lineterminator)
+
+
+
+

graalpython/com.oracle.graal.python.test/src/tests/unittest_tags/test_csv.txt

@@ -63,6 +63,7 @@
 *graalpython.lib-python.3.test.test_csv.TestDictFields.test_write_multiple_dict_rows
 *graalpython.lib-python.3.test.test_csv.TestDictFields.test_write_no_fields
 *graalpython.lib-python.3.test.test_csv.TestDictFields.test_write_simple_dict
+*graalpython.lib-python.3.test.test_csv.TestDictFields.test_writeheader_return_value
 *graalpython.lib-python.3.test.test_csv.TestEscapedExcel.test_escape_fieldsep
 *graalpython.lib-python.3.test.test_csv.TestEscapedExcel.test_read_escape_fieldsep
 *graalpython.lib-python.3.test.test_csv.TestLeaks.test_create_read
@@ -80,6 +81,7 @@
 *graalpython.lib-python.3.test.test_csv.TestUnicode.test_unicode_read
 *graalpython.lib-python.3.test.test_csv.TestUnicode.test_unicode_write
 *graalpython.lib-python.3.test.test_csv.Test_Csv.test_read_bigfield
+*graalpython.lib-python.3.test.test_csv.Test_Csv.test_read_eof
 *graalpython.lib-python.3.test.test_csv.Test_Csv.test_read_eol
 *graalpython.lib-python.3.test.test_csv.Test_Csv.test_read_escape
 *graalpython.lib-python.3.test.test_csv.Test_Csv.test_read_linenum
@@ -89,6 +91,7 @@
 *graalpython.lib-python.3.test.test_csv.Test_Csv.test_reader_attrs
 *graalpython.lib-python.3.test.test_csv.Test_Csv.test_reader_dialect_attrs
 *graalpython.lib-python.3.test.test_csv.Test_Csv.test_reader_kw_attrs
+*graalpython.lib-python.3.test.test_csv.Test_Csv.test_roundtrip_escaped_unquoted_newlines
 *graalpython.lib-python.3.test.test_csv.Test_Csv.test_roundtrip_quoteed_newlines
 *graalpython.lib-python.3.test.test_csv.Test_Csv.test_write_arg_valid
 *graalpython.lib-python.3.test.test_csv.Test_Csv.test_write_bigfield

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java

@@ -109,6 +109,10 @@
 import com.oracle.graal.python.builtins.modules.bz2.BZ2CompressorBuiltins;
 import com.oracle.graal.python.builtins.modules.bz2.BZ2DecompressorBuiltins;
 import com.oracle.graal.python.builtins.modules.bz2.BZ2ModuleBuiltins;
+import com.oracle.graal.python.builtins.modules.csv.CSVDialectBuiltins;
+import com.oracle.graal.python.builtins.modules.csv.CSVModuleBuiltins;
+import com.oracle.graal.python.builtins.modules.csv.CSVReaderBuiltins;
+import com.oracle.graal.python.builtins.modules.csv.CSVWriterBuiltins;
 import com.oracle.graal.python.builtins.modules.ctypes.CArgObjectBuiltins;
 import com.oracle.graal.python.builtins.modules.ctypes.CDataBuiltins;
 import com.oracle.graal.python.builtins.modules.ctypes.CDataTypeBuiltins;
@@ -482,6 +486,7 @@ private static PythonBuiltins[] initializeBuiltins(boolean nativeAccessAllowed)
                         new TupleGetterBuiltins(),
                         new JavaModuleBuiltins(),
                         new JArrayModuleBuiltins(),
+                        new CSVModuleBuiltins(),
                         new JSONModuleBuiltins(),
                         new SREModuleBuiltins(),
                         new AstModuleBuiltins(),
@@ -567,6 +572,11 @@ private static PythonBuiltins[] initializeBuiltins(boolean nativeAccessAllowed)
                         new JSONScannerBuiltins(),
                         new JSONEncoderBuiltins(),
 
+                        // csv
+                        new CSVDialectBuiltins(),
+                        new CSVReaderBuiltins(),
+                        new CSVWriterBuiltins(),
+
                         // _ast
                         new AstBuiltins(),
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/PythonBuiltinClassType.java

@@ -55,6 +55,7 @@
 import com.oracle.graal.python.builtins.objects.function.BuiltinMethodDescriptor;
 import com.oracle.graal.python.builtins.objects.type.SpecialMethodSlot;
 import com.oracle.graal.python.runtime.PythonContext;
+import com.oracle.graal.python.util.PythonUtils;
 import com.oracle.truffle.api.CompilerAsserts;
 import com.oracle.truffle.api.CompilerDirectives;
 import com.oracle.truffle.api.CompilerDirectives.CompilationFinal;
@@ -234,6 +235,11 @@ public enum PythonBuiltinClassType implements TruffleObject {
     JSONScanner("Scanner", "_json", Flags.PUBLIC_BASE_WODICT),
     JSONEncoder("Encoder", "_json", Flags.PUBLIC_BASE_WODICT),
 
+    // csv
+    CSVDialect("Dialect", "_csv", Flags.PUBLIC_BASE_WODICT),
+    CSVReader("Reader", "_csv", Flags.PUBLIC_BASE_WODICT),
+    CSVWriter("Writer", "_csv", Flags.PUBLIC_BASE_WODICT),
+
     // _ast (rest of the classes are not builtin, they are generated in AstModuleBuiltins)
     AST("AST", "_ast", Flags.PUBLIC_BASE_WDICT),
 
@@ -306,6 +312,7 @@ public enum PythonBuiltinClassType implements TruffleObject {
     TimeoutError("TimeoutError", BUILTINS, Flags.EXCEPTION),
     ZipImportError("ZipImportError", "zipimport", Flags.EXCEPTION),
     ZLibError("error", "zlib", Flags.EXCEPTION),
+    CSVError("Error", "_csv", Flags.EXCEPTION),
     LZMAError("LZMAError", "_lzma", Flags.EXCEPTION),
     StructError("StructError", "_struct", Flags.EXCEPTION),
     PickleError("PickleError", "_pickle", Flags.EXCEPTION),
@@ -601,6 +608,7 @@ public final Shape getInstanceShape(PythonLanguage lang) {
         TimeoutError.base = OSError;
         ZipImportError.base = ImportError;
         ZLibError.base = Exception;
+        CSVError.base = Exception;
         LZMAError.base = Exception;
         SocketGAIError.base = OSError;
         SocketHError.base = OSError;
@@ -709,9 +717,10 @@ public final Shape getInstanceShape(PythonLanguage lang) {
 
         Empty.base = Exception;
 
-        HashSet<String> set = new HashSet<>();
+        HashSet<String> set = PythonUtils.ASSERTIONS_ENABLED ? new HashSet<>() : null;
         for (PythonBuiltinClassType type : VALUES) {
-            assert set.add(type.name) : type.name(); // check uniqueness
+            // check uniqueness
+            assert set.add("" + type.moduleName + "." + type.name) : type.name();
 
             /* Initialize type.base (defaults to PythonObject unless that's us) */
             if (type.base == null && type != PythonObject) {

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/csv/CSVDialect.java

@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package com.oracle.graal.python.builtins.modules.csv;
+
+import static com.oracle.graal.python.builtins.modules.csv.CSVModuleBuiltins.NOT_SET;
+import static com.oracle.graal.python.builtins.modules.csv.CSVModuleBuiltins.NOT_SET_CODEPOINT;
+
+import com.oracle.graal.python.builtins.objects.object.PythonBuiltinObject;
+import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
+import com.oracle.truffle.api.object.Shape;
+
+public final class CSVDialect extends PythonBuiltinObject {
+    String delimiter; /* field separator */
+    boolean doubleQuote; /* is " represented by ""? */
+    String escapeChar; /* escape character */
+    String lineTerminator; /* string to write between records */
+    QuoteStyle quoting; /* style of quoting to write */
+    String quoteChar; /* quote character */
+    boolean skipInitialSpace; /* ignore spaces following delimiter? */
+    boolean strict; /* raise exception on bad CSV */
+
+    int delimiterCodePoint; /* code point representation for handling utf-32 delimiters */
+    int escapeCharCodePoint; /* code point representation for handling utf-32 escape chars */
+    int quoteCharCodePoint; /* code point representation for handling utf-32 quote chars */
+    int[] lineTerminatorCodePoints; /*
+                                     * code point representation for handling utf-32 chars in line
+                                     * terminator
+                                     */
+
+    public CSVDialect(Object cls, Shape instanceShape) {
+        super(cls, instanceShape);
+    }
+
+    public CSVDialect(Object cls, Shape instanceShape, String delimiter, boolean doubleQuote, String escapeChar,
+                    String lineTerminator, String quoteChar, QuoteStyle quoting, boolean skipInitialSpace,
+                    boolean strict) {
+        super(cls, instanceShape);
+        this.delimiter = delimiter;
+        this.doubleQuote = doubleQuote;
+        this.escapeChar = escapeChar;
+        this.lineTerminator = lineTerminator;
+        this.quoteChar = quoteChar;
+        this.quoting = quoting;
+        this.skipInitialSpace = skipInitialSpace;
+        this.strict = strict;
+
+        this.delimiterCodePoint = this.delimiter.codePointAt(0); // delimiter cannot be NOT_SET
+        this.escapeCharCodePoint = this.escapeChar.equals(NOT_SET) ? NOT_SET_CODEPOINT : this.escapeChar.codePointAt(0);
+        this.quoteCharCodePoint = this.quoteChar.equals(NOT_SET) ? NOT_SET_CODEPOINT : this.quoteChar.codePointAt(0);
+        this.lineTerminatorCodePoints = strToCodePointArray(this.lineTerminator);
+    }
+
+    @TruffleBoundary
+    private static int[] strToCodePointArray(String str) {
+        final int strLen = str.length();
+        final int codePointCount = str.codePointCount(0, strLen);
+        int[] codePoints = new int[codePointCount];
+
+        for (int offset = 0, index = 0; offset < strLen; index++) {
+            final int c = str.codePointAt(offset);
+            codePoints[index] = c;
+            offset += Character.charCount(c);
+        }
+        return codePoints;
+    }
+}