Add 'test_mmap' and make a basic set of tests pass.

Author: fangerer

graalpython/com.oracle.graal.python.test/src/tests/test_mmap.py

@@ -0,0 +1,184 @@
+from test.support import (TESTFN, run_unittest, import_module, unlink,
+                          requires, _2G, _4G, gc_collect, cpython_only)
+import unittest
+import os
+import re
+import itertools
+import socket
+import sys
+import weakref
+
+# Skip test if we can't import mmap.
+mmap = import_module('mmap')
+
+PAGESIZE = mmap.PAGESIZE
+
+class MmapTests(unittest.TestCase):
+
+    def setUp(self):
+        if os.path.exists(TESTFN):
+            os.unlink(TESTFN)
+
+    def tearDown(self):
+        try:
+            os.unlink(TESTFN)
+        except OSError:
+            pass
+
+    def test_basic(self):
+        # Test mmap module on Unix systems and Windows
+
+        # Create a file to be mmap'ed.
+        f = open(TESTFN, 'bw+')
+        try:
+            # Write 2 pages worth of data to the file
+            f.write(b'\0'* PAGESIZE)
+            f.write(b'foo')
+            f.write(b'\0'* (PAGESIZE-3) )
+            f.flush()
+            m = mmap.mmap(f.fileno(), 2 * PAGESIZE)
+        finally:
+            f.close()
+
+        # Simple sanity checks
+
+        tp = str(type(m))  # SF bug 128713:  segfaulted on Linux
+        self.assertEqual(m.find(b'foo'), PAGESIZE)
+
+        self.assertEqual(len(m), 2*PAGESIZE)
+
+        self.assertEqual(m[0], 0)
+        self.assertEqual(m[0:3], b'\0\0\0')
+
+        # Shouldn't crash on boundary (Issue #5292)
+        self.assertRaises(IndexError, m.__getitem__, len(m))
+        self.assertRaises(IndexError, m.__setitem__, len(m), b'\0')
+
+        # Modify the file's content
+        m[0] = b'3'[0]
+        m[PAGESIZE +3: PAGESIZE +3+3] = b'bar'
+
+        # Check that the modification worked
+        self.assertEqual(m[0], b'3'[0])
+        self.assertEqual(m[0:3], b'3\0\0')
+        self.assertEqual(m[PAGESIZE-1 : PAGESIZE + 7], b'\0foobar\0')
+
+        m.flush()
+
+        # Test doing a regular expression match in an mmap'ed file
+        match = re.search(b'[A-Za-z]+', m)
+        if match is None:
+            self.fail('regex match on mmap failed!')
+        else:
+            start, end = match.span(0)
+            length = end - start
+
+            self.assertEqual(start, PAGESIZE)
+            self.assertEqual(end, PAGESIZE + 6)
+
+        # test seeking around (try to overflow the seek implementation)
+        m.seek(0,0)
+        self.assertEqual(m.tell(), 0)
+        m.seek(42,1)
+        self.assertEqual(m.tell(), 42)
+        m.seek(0,2)
+        self.assertEqual(m.tell(), len(m))
+
+        # Try to seek to negative position...
+        self.assertRaises(ValueError, m.seek, -1)
+
+        # Try to seek beyond end of mmap...
+        self.assertRaises(ValueError, m.seek, 1, 2)
+
+        # Try to seek to negative position...
+        self.assertRaises(ValueError, m.seek, -len(m)-1, 2)
+
+        # Try resizing map
+#         try:
+#             m.resize(512)
+#         except SystemError:
+#             # resize() not supported
+#             # No messages are printed, since the output of this test suite
+#             # would then be different across platforms.
+#             pass
+#         else:
+#             # resize() is supported
+#             self.assertEqual(len(m), 512)
+#             # Check that we can no longer seek beyond the new size.
+#             self.assertRaises(ValueError, m.seek, 513, 0)
+# 
+#             # Check that the underlying file is truncated too
+#             # (bug #728515)
+#             f = open(TESTFN, 'rb')
+#             try:
+#                 f.seek(0, 2)
+#                 self.assertEqual(f.tell(), 512)
+#             finally:
+#                 f.close()
+#             self.assertEqual(m.size(), 512)
+
+        m.close()
+
+
+    def test_bad_file_desc(self):
+        # Try opening a bad file descriptor...
+        self.assertRaises(OSError, mmap.mmap, -2, 4096)
+
+
+    def test_anonymous(self):
+        # anonymous mmap.mmap(-1, PAGE)
+        m = mmap.mmap(-1, PAGESIZE)
+        for x in range(PAGESIZE):
+            self.assertEqual(m[x], 0,
+                             "anonymously mmap'ed contents should be zero")
+
+        for x in range(PAGESIZE):
+            b = x & 0xff
+            m[x] = b
+            self.assertEqual(m[x], b)
+
+    def test_read_all(self):
+        m = mmap.mmap(-1, 16)
+
+        # With no parameters, or None or a negative argument, reads all
+        m.write(bytes(range(16)))
+        m.seek(0)
+        self.assertEqual(m.read(), bytes(range(16)))
+        m.seek(8)
+        self.assertEqual(m.read(), bytes(range(8, 16)))
+        m.seek(16)
+        self.assertEqual(m.read(), b'')
+        m.seek(3)
+        self.assertEqual(m.read(None), bytes(range(3, 16)))
+        m.seek(4)
+        self.assertEqual(m.read(-1), bytes(range(4, 16)))
+        m.seek(5)
+        self.assertEqual(m.read(-2), bytes(range(5, 16)))
+        m.seek(9)
+        self.assertEqual(m.read(-42), bytes(range(9, 16)))
+        m.close()
+
+
+
+    def test_context_manager(self):
+        with mmap.mmap(-1, 10) as m:
+            self.assertFalse(m.closed)
+        self.assertTrue(m.closed)
+
+    def test_context_manager_exception(self):
+        # Test that the OSError gets passed through
+        with self.assertRaises(Exception) as exc:
+            with mmap.mmap(-1, 10) as m:
+                raise OSError
+        self.assertIsInstance(exc.exception, OSError,
+                              "wrong exception raised in context manager")
+        self.assertTrue(m.closed, "context manager failed")
+
+
+
+def test_main():
+    #run_unittest(MmapTests, LargeMmapTests)
+    run_unittest(MmapTests)
+
+if __name__ == '__main__':
+    test_main()

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/MMapModuleBuiltins.java

@@ -44,9 +44,8 @@
 import static com.oracle.graal.python.builtins.PythonBuiltinClassType.ValueError;
 
 import java.io.IOException;
-import java.nio.channels.FileChannel.MapMode;
 import java.nio.ByteBuffer;
-import java.nio.channels.ByteChannel;
+import java.nio.channels.FileChannel.MapMode;
 import java.nio.channels.SeekableByteChannel;
 import java.nio.file.StandardOpenOption;
 import java.util.HashSet;
@@ -67,7 +66,6 @@
 import com.oracle.truffle.api.dsl.NodeFactory;
 import com.oracle.truffle.api.dsl.Specialization;
 import com.oracle.truffle.api.profiles.BranchProfile;
-import com.oracle.truffle.api.profiles.ValueProfile;
 
 @CoreFunctions(defineModule = "mmap")
 public class MMapModuleBuiltins extends PythonBuiltins {
@@ -92,13 +90,13 @@ public MMapModuleBuiltins() {
     @GenerateNodeFactory
     public abstract static class MMapNode extends PythonBuiltinNode {
 
-        private final ValueProfile classProfile = ValueProfile.createClassProfile();
         private final BranchProfile invalidLengthProfile = BranchProfile.create();
 
-        @Specialization(guards = {"fd < 0", "isNoValue(access)", "isNoValue(offset)"})
-        PMMap doAnonymous(LazyPythonClass clazz, int fd, int length, Object tagname, @SuppressWarnings("unused") PNone access, @SuppressWarnings("unused") PNone offset) {
+        @Specialization(guards = {"isAnonymous(fd)", "isNoValue(access)", "isNoValue(offset)"})
+        PMMap doAnonymous(LazyPythonClass clazz, @SuppressWarnings("unused") int fd, int length, @SuppressWarnings("unused") Object tagname, @SuppressWarnings("unused") PNone access,
+                        @SuppressWarnings("unused") PNone offset) {
             checkLength(length);
-            return new PMMap(clazz, new AnonymousMap(length), length, 0);
+            return factory().createMMap(clazz, new AnonymousMap(length), length, 0);
         }
 
         @Specialization(guards = {"fd >= 0", "isNoValue(access)", "isNoValue(offset)"})
@@ -108,14 +106,13 @@ PMMap doIt(LazyPythonClass clazz, int fd, int length, Object tagname, @SuppressW
 
         // mmap(fileno, length, tagname=None, access=ACCESS_DEFAULT[, offset])
         @Specialization(guards = "fd >= 0")
-        PMMap doFile(LazyPythonClass clazz, int fd, int length, @SuppressWarnings("unused") Object tagname, int access, long offset) {
+        PMMap doFile(LazyPythonClass clazz, int fd, int length, @SuppressWarnings("unused") Object tagname, @SuppressWarnings("unused") int access, long offset) {
             checkLength(length);
 
             String path = getContext().getResources().getFilePath(fd);
             TruffleFile truffleFile = getContext().getEnv().getTruffleFile(path);
 
             // TODO(fa) correctly honor access flags
-            MapMode mode = convertAccessToMapMode(access);
             Set<StandardOpenOption> options = new HashSet<>();
             options.add(StandardOpenOption.READ);
             options.add(StandardOpenOption.WRITE);
@@ -125,12 +122,27 @@ PMMap doFile(LazyPythonClass clazz, int fd, int length, @SuppressWarnings("unuse
             try {
                 fileChannel = truffleFile.newByteChannel(options);
                 fileChannel.position(offset);
-                return new PMMap(PythonBuiltinClassType.PMMap, fileChannel, length, offset);
+                return factory().createMMap(clazz, fileChannel, length, offset);
             } catch (IOException e) {
                 throw raise(ValueError, "cannot mmap file");
             }
         }
 
+        @Specialization(guards = "isIllegal(fd)")
+        @SuppressWarnings("unused")
+        PMMap doAnonymous(LazyPythonClass clazz, int fd, Object length, Object tagname, PNone access, PNone offset) {
+            throw raise(PythonBuiltinClassType.OSError);
+        }
+
+        protected static boolean isAnonymous(int fd) {
+            return fd == -1;
+        }
+
+        protected static boolean isIllegal(int fd) {
+            return fd < -1;
+        }
+
+        @SuppressWarnings("unused")
         private MapMode convertAccessToMapMode(int access) {
             switch (access) {
                 case 0:
@@ -189,7 +201,7 @@ public long position() throws IOException {
         }
 
         public SeekableByteChannel position(long newPosition) throws IOException {
-            if (newPosition < 0 || newPosition >= data.length) {
+            if (newPosition < 0) {
                 throw new IllegalArgumentException();
             }
             cur = (int) newPosition;

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/mmap/MMapBuiltins.java

@@ -246,7 +246,7 @@ int doSingle(VirtualFrame frame, PMMap self, Object idxObj,
                 long oldPos = channel.position();
 
                 channel.position(idx);
-                int res = readByteNode.execute(channel);
+                int res = readByteNode.execute(channel) & 0xFF;
 
                 // restore position
                 channel.position(oldPos);
@@ -458,18 +458,25 @@ int readByte(PMMap self,
     @TypeSystemReference(PythonArithmeticTypes.class)
     abstract static class ReadNode extends PythonBuiltinNode {
 
-        @Specialization(guards = "isNoValue(n)")
-        PBytes read(PMMap self, @SuppressWarnings("unused") PNone n,
+        @Specialization
+        PBytes readUnlimited(PMMap self, @SuppressWarnings("unused") PNone n,
                         @Cached("create()") ReadFromChannelNode readChannelNode) {
+            // intentionally accept NO_VALUE and NONE; both mean that we read unlimited amount of
+            // bytes
             ByteSequenceStorage res = readChannelNode.execute(self.getChannel(), ReadFromChannelNode.MAX_READ);
             return factory().createBytes(res);
         }
 
         @Specialization(guards = "!isNoValue(n)")
         PBytes read(PMMap self, Object n,
                         @Cached("create()") ReadFromChannelNode readChannelNode,
-                        @Cached("create()") CastToIndexNode castToIndexNode) {
-            ByteSequenceStorage res = readChannelNode.execute(self.getChannel(), castToIndexNode.execute(n));
+                        @Cached("create()") CastToIndexNode castToIndexNode,
+                        @Cached("createBinaryProfile()") ConditionProfile negativeProfile) {
+            int nread = castToIndexNode.execute(n);
+            if (negativeProfile.profile(nread < 0)) {
+                return readUnlimited(self, PNone.NO_VALUE, readChannelNode);
+            }
+            ByteSequenceStorage res = readChannelNode.execute(self.getChannel(), nread);
             return factory().createBytes(res);
         }
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/util/ChannelNodes.java

@@ -178,7 +178,7 @@ int readByte(ReadableByteChannel channel,
             if (readProfile.profile(read != 1)) {
                 return handleError(channel);
             }
-            return get(buf);
+            return get(buf) & 0xFF;
         }
 
         private int handleError(Channel channel) {