[GR-28392] More fixes for sockets

PullRequest: graalpython/1603

Author: msimacek

graalpython/com.oracle.graal.python.test/src/graalpytest.py

@@ -577,10 +577,12 @@ def run(cls, items=None):
                 return instance
         for k, v in items:
             if k.startswith("test"):
+                testfn = getattr(instance, k, v)
                 if patterns:
-                    if not any(p in k for p in patterns):
+                    import fnmatch
+                    if not any(fnmatch.fnmatch(testfn.__qualname__, p) for p in patterns):
                         continue
-                instance.run_test(getattr(instance, k, v))
+                instance.run_test(testfn)
         if hasattr(instance, "tearDownClass"):
             instance.run_safely(instance.tearDownClass)
         return instance
@@ -752,7 +754,10 @@ class TextTestResult():
         if argv[idx] == "-k":
             argv.pop(idx)
             try:
-                patterns.append(argv.pop(idx))
+                pattern = argv.pop(idx)
+                if '*' not in pattern:
+                    pattern = '*' + pattern + '*'
+                patterns.append(pattern)
             except IndexError:
                 print("-k needs an argument")
         else:

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/socket/SocketBuiltins.java

@@ -48,7 +48,6 @@
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
-import java.net.SocketAddress;
 import java.nio.ByteBuffer;
 import java.nio.channels.NotYetConnectedException;
 import java.nio.channels.ServerSocketChannel;
@@ -82,6 +81,7 @@
 import com.oracle.graal.python.nodes.function.builtins.PythonBinaryBuiltinNode;
 import com.oracle.graal.python.nodes.function.builtins.PythonBinaryClinicBuiltinNode;
 import com.oracle.graal.python.nodes.function.builtins.PythonTernaryBuiltinNode;
+import com.oracle.graal.python.nodes.function.builtins.PythonTernaryClinicBuiltinNode;
 import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode;
 import com.oracle.graal.python.nodes.function.builtins.clinic.ArgumentClinicProvider;
 import com.oracle.graal.python.nodes.util.CannotCastException;
@@ -129,16 +129,18 @@ Object accept(PSocket socket) {
                 if (acceptSocket == null) {
                     throw raiseOSError(null, OSErrorEnum.EWOULDBLOCK);
                 }
-                SocketAddress addr = acceptSocket.getLocalAddress();
-                if (!acceptSocket.socket().isBound() || addr == null) {
+                InetSocketAddress remoteAddress = (InetSocketAddress) acceptSocket.getRemoteAddress();
+                if (!acceptSocket.socket().isBound() || remoteAddress == null) {
                     throw raise(OSError);
                 }
                 PSocket newSocket = factory().createSocket(socket.getFamily(), socket.getType(), socket.getProto());
                 int fd = getContext().getResources().openSocket(newSocket);
                 newSocket.setFileno(fd);
                 newSocket.setSocket(acceptSocket);
-                Object[] output = {fd, ((InetSocketAddress) addr).getAddress().getHostAddress()};
-                return factory().createTuple(output);
+                SocketUtils.setBlocking(newSocket, socket.isBlocking());
+                newSocket.setTimeout(socket.getTimeout());
+                PTuple addressTuple = factory().createTuple(new Object[]{remoteAddress.getAddress().getHostAddress(), remoteAddress.getPort()});
+                return factory().createTuple(new Object[]{fd, addressTuple});
             } catch (IOException e) {
                 throw raise(OSError);
             }
@@ -174,20 +176,12 @@ abstract static class CloseNode extends PythonUnaryBuiltinNode {
         @TruffleBoundary
         Object close(PSocket socket) {
             if (socket.getSocket() != null) {
-                if (!socket.getSocket().isOpen()) {
-                    throw raise(OSError, ErrorMessages.BAD_FILE_DESCRIPTOR);
-                }
-
                 try {
                     socket.getSocket().close();
                 } catch (IOException e) {
                     throw raise(OSError, ErrorMessages.BAD_FILE_DESCRIPTOR);
                 }
             } else if (socket.getServerSocket() != null) {
-                if (!socket.getServerSocket().isOpen()) {
-                    throw raise(OSError, ErrorMessages.BAD_FILE_DESCRIPTOR);
-                }
-
                 try {
                     socket.getServerSocket().close();
                 } catch (IOException e) {
@@ -337,9 +331,11 @@ Object listen(PSocket socket, PNone backlog) {
     }
 
     // recv(bufsize[, flags])
-    @Builtin(name = "recv", minNumOfPositionalArgs = 2, maxNumOfPositionalArgs = 3)
+    @Builtin(name = "recv", minNumOfPositionalArgs = 2, numOfPositionalOnlyArgs = 3, parameterNames = {"$self", "nbytes", "flags"})
+    @ArgumentClinic(name = "nbytes", conversion = ArgumentClinic.ClinicConversion.Index)
+    @ArgumentClinic(name = "flags", conversion = ArgumentClinic.ClinicConversion.Int, defaultValue = "0")
     @GenerateNodeFactory
-    abstract static class RecvNode extends PythonTernaryBuiltinNode {
+    abstract static class RecvNode extends PythonTernaryClinicBuiltinNode {
         @Specialization
         Object recv(VirtualFrame frame, PSocket socket, int bufsize, int flags) {
             ByteBuffer readBytes = PythonUtils.allocateByteBuffer(bufsize);
@@ -353,6 +349,10 @@ Object recv(VirtualFrame frame, PSocket socket, int bufsize, int flags) {
             }
         }
 
+        @Override
+        protected ArgumentClinicProvider getArgumentClinic() {
+            return SocketBuiltinsClinicProviders.RecvNodeClinicProviderGen.INSTANCE;
+        }
     }
 
     // recvfrom(bufsize[, flags])

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/EmulatedPosixSupport.java

@@ -90,7 +90,6 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
-import java.util.function.Function;
 import java.util.logging.Level;
 
 import org.graalvm.nativeimage.ImageInfo;
@@ -425,9 +424,24 @@ public int[] pipeMessage() throws PosixException {
     public SelectResult select(int[] readfds, int[] writefds, int[] errorfds, Timeval timeout) throws PosixException {
         SelectableChannel[] readChannels = getSelectableChannels(readfds);
         SelectableChannel[] writeChannels = getSelectableChannels(writefds);
-        SelectableChannel[] errChannels = getSelectableChannels(errorfds);
+        // Java doesn't support any exceptional conditions we could apply on errfds, report a
+        // warning if errfds is not a subset of readfds & writefds
+        errfdsCheck: for (int fd : errorfds) {
+            for (int fd2 : readfds) {
+                if (fd == fd2) {
+                    continue errfdsCheck;
+                }
+            }
+            for (int fd2 : writefds) {
+                if (fd == fd2) {
+                    continue errfdsCheck;
+                }
+            }
+            compatibilityIgnored("POSIX emultaion layer doesn't support waiting on exceptional conditions in select()");
+            break;
+        }
 
-        boolean[] wasBlocking = new boolean[readChannels.length + writeChannels.length + errChannels.length];
+        boolean[] wasBlocking = new boolean[readChannels.length + writeChannels.length];
         int i = 0;
 
         for (SelectableChannel channel : readChannels) {
@@ -436,25 +450,19 @@ public SelectResult select(int[] readfds, int[] writefds, int[] errorfds, Timeva
         for (SelectableChannel channel : writeChannels) {
             wasBlocking[i++] = channel.isBlocking();
         }
-        for (SelectableChannel channel : errChannels) {
-            wasBlocking[i++] = channel.isBlocking();
-        }
+
+        final int readOps = SelectionKey.OP_READ | SelectionKey.OP_ACCEPT;
+        final int writeOps = SelectionKey.OP_WRITE;
 
         try (Selector selector = Selector.open()) {
             for (SelectableChannel channel : readChannels) {
                 channel.configureBlocking(false);
-                channel.register(selector, (SelectionKey.OP_READ | SelectionKey.OP_ACCEPT) & channel.validOps());
+                channel.register(selector, readOps & channel.validOps());
             }
 
             for (SelectableChannel channel : writeChannels) {
                 channel.configureBlocking(false);
-                channel.register(selector, SelectionKey.OP_WRITE);
-            }
-
-            for (SelectableChannel channel : errChannels) {
-                // TODO(fa): not sure if these ops are representing "exceptional condition pending"
-                channel.configureBlocking(false);
-                channel.register(selector, SelectionKey.OP_CONNECT);
+                channel.register(selector, writeOps);
             }
 
             // IMPORTANT: The meaning of the timeout value is slightly different: 'timeout == 0.0'
@@ -478,9 +486,9 @@ public SelectResult select(int[] readfds, int[] writefds, int[] errorfds, Timeva
             int selected = useSelectNow ? selector.selectNow() : selector.select(timeoutMs);
 
             // remove non-selected channels from given lists
-            boolean[] resReadfds = createSelectedMap(readfds, readChannels, selector, SelectionKey::isReadable);
-            boolean[] resWritefds = createSelectedMap(writefds, writeChannels, selector, SelectionKey::isWritable);
-            boolean[] resErrfds = createSelectedMap(errorfds, errChannels, selector, key -> key.isAcceptable() || key.isConnectable());
+            boolean[] resReadfds = createSelectedMap(readfds, readChannels, selector, readOps);
+            boolean[] resWritefds = createSelectedMap(writefds, writeChannels, selector, writeOps);
+            boolean[] resErrfds = new boolean[errorfds.length];
 
             assert selected == countSelected(resReadfds) + countSelected(resWritefds) + countSelected(resErrfds);
             return new SelectResult(resReadfds, resWritefds, resErrfds);
@@ -499,23 +507,18 @@ public SelectResult select(int[] readfds, int[] writefds, int[] errorfds, Timeva
                         channel.configureBlocking(true);
                     }
                 }
-                for (SelectableChannel channel : errChannels) {
-                    if (wasBlocking[i++]) {
-                        channel.configureBlocking(true);
-                    }
-                }
             } catch (IOException e) {
                 // We didn't manage to restore the blocking status, ignore
             }
         }
     }
 
-    private static boolean[] createSelectedMap(int[] fds, SelectableChannel[] channels, Selector selector, Function<SelectionKey, Boolean> selectedPredicate) {
+    private static boolean[] createSelectedMap(int[] fds, SelectableChannel[] channels, Selector selector, int op) {
         boolean[] result = new boolean[fds.length];
         for (int i = 0; i < channels.length; i++) {
             SelectableChannel channel = channels[i];
             SelectionKey selectionKey = channel.keyFor(selector);
-            result[i] = selectedPredicate.apply(selectionKey);
+            result[i] = (selectionKey.readyOps() & op) != 0;
         }
         return result;
     }

mx.graalpython/mx_graalpython.py

@@ -523,8 +523,8 @@ def _hpy_test_root():
 def graalpytest(args):
     parser = ArgumentParser(prog='mx graalpytest')
     parser.add_argument('--python', type=str, action='store', default="", help='Run tests with custom Python binary.')
-    parser.add_argument('-v', "--verbose", action="store_true", help='Verbose output.')
-    parser.add_argument('-k', dest="filter", default=[], help='Test pattern.')
+    parser.add_argument('-v', "--verbose", action="store_true", help='Verbose output.', default=True)
+    parser.add_argument('-k', dest="filter", default='', help='Test pattern.')
     parser.add_argument('test', nargs="*", default=[], help='Test file to run (specify absolute or relative; e.g. "/path/to/test_file.py" or "cpyext/test_object.py") ')
     args, unknown_args = parser.parse_known_args(args)
 

mx.graalpython/mx_graalpython_bisect.py

@@ -204,6 +204,7 @@ def _bisect_benchmark(argv, initial_branch, email_to):
     def benchmark_callback(suite, commit):
         suite.vc.update_to_branch(suite.vc_dir, commit)
         mx.run_mx(['sforceimports'], suite=suite)
+        mx.run_mx(['--env', 'ce', 'sforceimports'], suite=get_suite('/vm'))
         if args.enterprise and suite.name != 'vm-enterprise':
             checkout_args = ['--dynamicimports', '/vm-enterprise', 'checkout-downstream', 'vm', 'vm-enterprise']
             if fetched_enterprise[0]:
@@ -215,8 +216,11 @@ def benchmark_callback(suite, commit):
             fetched_enterprise[0] = True
         suite.vc.update_to_branch(suite.vc_dir, commit)
         mx.run_mx(['sforceimports'], suite=suite)
-        print("debug: graalpython={} graal={} graal-enterprise={}"
-              .format(*(get_commit(get_suite(s)) for s in ('graalpython', '/vm', '/vm-enterprise'))))
+        debug_str = "debug: graalpython={} graal={}".format(
+            get_commit(get_suite('graalpython')), get_commit(get_suite('/vm')))
+        if args.enterprise:
+            debug_str += " graal-enterprise={}".format(get_commit(get_suite('/vm-enterprise')))
+        print(debug_str)
         env = os.environ.copy()
         env['MX_ALT_OUTPUT_ROOT'] = 'mxbuild-{}'.format(commit)
         retcode = mx.run(shlex.split(args.build_command), env=env, nonZeroIsFatal=False)