add a per context shutdown hook mechanism

- needed for removing temporary files that are managed withing the python context

- add os.ftruncate

- add os.dup2

- add faulthandler.disable()

- add faulthandler _enabled flag

Author: cosminbasca

graalpython/com.oracle.graal.python.test/src/tests/test_tempfile.py

@@ -53,6 +53,7 @@ def test_create():
         f = TemporaryFile()
         assert f is not None
         f.close()
-    except:
+    except Exception as e:
+        print(e)
         assert False
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java

@@ -153,6 +153,7 @@ public NodeFactory getNodeFactory() {
     @Override
     protected void finalizeContext(PythonContext context) {
         context.runShutdownHooks();
+        context.getResources().release();
         super.finalizeContext(context);
     }
 

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java

@@ -214,6 +214,7 @@ private static final String[] initializeCoreFiles() {
                         "_weakref",
                         "set",
                         "itertools",
+                        "faulthandler",
                         "base_exception",
                         PythonCextBuiltins.PYTHON_CEXT,
                         "_collections",

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixModuleBuiltins.java

@@ -142,6 +142,7 @@
 import com.oracle.graal.python.runtime.exception.PythonExitException;
 import com.oracle.graal.python.runtime.sequence.PSequence;
 import com.oracle.graal.python.runtime.sequence.storage.ByteSequenceStorage;
+import com.oracle.graal.python.util.FileDeleteShutdownHook;
 import com.oracle.truffle.api.CompilerDirectives;
 import com.oracle.truffle.api.CompilerDirectives.CompilationFinal;
 import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
@@ -903,6 +904,30 @@ int dupOvf(PInt fd) {
         }
     }
 
+    @Builtin(name = "dup2", minNumOfPositionalArgs = 2)
+    @GenerateNodeFactory
+    @TypeSystemReference(PythonArithmeticTypes.class)
+    abstract static class Dup2Node extends PythonFileNode {
+        @Specialization
+        int dup(int fd, int fd2) {
+            return getResources().dup2(fd, fd2);
+        }
+
+        @Specialization(rewriteOn = ArithmeticException.class)
+        int dupPInt(PInt fd, PInt fd2) {
+            return getResources().dup2(fd.intValueExact(), fd2.intValueExact());
+        }
+
+        @Specialization(replaces = "dupPInt")
+        int dupOvf(PInt fd, PInt fd2) {
+            try {
+                return dupPInt(fd, fd2);
+            } catch (ArithmeticException e) {
+                throw raise(OSError, "invalid fd %r", fd);
+            }
+        }
+    }
+
     @Builtin(name = "open", minNumOfPositionalArgs = 2, parameterNames = {"pathname", "flags", "mode", "dir_fd"})
     @GenerateNodeFactory
     @TypeSystemReference(PythonArithmeticTypes.class)
@@ -929,7 +954,8 @@ Object openMode(VirtualFrame frame, Object pathArg, int flags, int fileMode, @Su
                 if (options.contains(StandardOpenOption.DELETE_ON_CLOSE)) {
                     // TODO: (cbas) remove patch once the TruffleFile API supports temp file
                     // creation -> GR-17515
-                    pathname = getRandomFilePath(pathname, options);
+                    pathname = getRandomFilePath(pathname, options, getContext().getEnv());
+                    getContext().registerShutdownHook(new FileDeleteShutdownHook(pathname));
                 }
                 truffleFile = getContext().getPublicTruffleFileRelaxed(pathname, PythonLanguage.DEFAULT_PYTHON_EXTENSIONS);
                 fc = truffleFile.newByteChannel(options, attributes);
@@ -1060,12 +1086,6 @@ Object close(Object fdObject,
                 throw raise(OSError, "invalid fd");
             } else {
                 resources.close(fd);
-                try {
-                    closeChannel(channel);
-                } catch (IOException e) {
-                    gotException.enter();
-                    throw raise(OSError, e);
-                }
             }
             return PNone.NONE;
         }
@@ -1981,4 +2001,18 @@ PNone fsync(VirtualFrame frame, int fd) {
             return PNone.NONE;
         }
     }
+
+    @Builtin(name = "ftruncate", minNumOfPositionalArgs = 2)
+    @GenerateNodeFactory
+    abstract static class FTruncateNode extends PythonBinaryBuiltinNode {
+        @Specialization
+        PNone ftruncate(VirtualFrame frame, int fd, long length) {
+            try {
+                getContext().getResources().ftruncate(fd, length);
+            } catch (IOException e) {
+                throw raiseOSError(frame, OSErrorEnum.ENOENT.getNumber());
+            }
+            return PNone.NONE;
+        }
+    }
 }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PosixResources.java

@@ -46,12 +46,15 @@
 import java.nio.channels.Channel;
 import java.nio.channels.Channels;
 import java.nio.channels.Pipe;
+import java.nio.channels.SeekableByteChannel;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 
 import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
 import com.oracle.truffle.api.TruffleFile;
@@ -68,6 +71,7 @@
 public class PosixResources {
     /** Context-local file-descriptor mappings and PID mappings */
     private final List<Channel> files;
+    private final Set<Channel> closedFiles;
     private final List<String> filePaths;
     private final List<Process> children;
     private final Map<String, Integer> inodes;
@@ -137,6 +141,7 @@ public Process destroyForcibly() {
 
     public PosixResources() {
         files = Collections.synchronizedList(new ArrayList<>());
+        closedFiles = Collections.synchronizedSet(new HashSet<>());
         filePaths = Collections.synchronizedList(new ArrayList<>());
         children = Collections.synchronizedList(new ArrayList<>());
         String osProperty = System.getProperty("os.name");
@@ -184,6 +189,7 @@ public String getFilePath(int fd) {
     @TruffleBoundary(allowInlining = true)
     public void close(int fd) {
         if (filePaths.size() > fd) {
+            closedFiles.add(files.get(fd));
             files.set(fd, null);
             filePaths.set(fd, null);
         }
@@ -210,10 +216,28 @@ public int dup(int fd) {
         return dupFd;
     }
 
+    @TruffleBoundary(allowInlining = true)
+    public int dup2(int fd, int fd2) {
+        int dupFd = nextFreeFd();
+        files.set(dupFd, getFileChannel(fd));
+        filePaths.set(dupFd, getFilePath(fd));
+        return dupFd;
+    }
+
     public void fsync(int fd) throws ArrayIndexOutOfBoundsException {
         files.get(fd); // for the side-effect
     }
 
+    @TruffleBoundary(allowInlining = true)
+    public void ftruncate(int fd, long size) throws IOException {
+        if (filePaths.size() > fd) {
+            Channel channel = files.get(fd);
+            if (channel instanceof SeekableByteChannel) {
+                ((SeekableByteChannel) channel).truncate(size);
+            }
+        }
+    }
+
     @TruffleBoundary(allowInlining = true)
     public int[] pipe() throws IOException {
         Pipe pipe = Pipe.open();
@@ -316,4 +340,15 @@ public int getInodeId(String canonical) {
             return inodeId;
         }
     }
+
+    @TruffleBoundary
+    public void release() {
+        // close all channels that have not been closed
+        for (Channel channel: closedFiles) {
+            try {
+                channel.close();
+            } catch (IOException ignored) {}
+        }
+        closedFiles.clear();
+    }
 }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java

@@ -36,7 +36,9 @@
 import java.io.OutputStream;
 import java.nio.file.LinkOption;
 import java.text.MessageFormat;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.function.Supplier;
@@ -56,6 +58,7 @@
 import com.oracle.graal.python.nodes.SpecialAttributeNames;
 import com.oracle.graal.python.runtime.AsyncHandler.AsyncAction;
 import com.oracle.graal.python.runtime.exception.PException;
+import com.oracle.graal.python.util.ShutdownHook;
 import com.oracle.truffle.api.Assumption;
 import com.oracle.truffle.api.CallTarget;
 import com.oracle.truffle.api.CompilerDirectives;
@@ -86,6 +89,7 @@ public final class PythonContext {
     private final PythonLanguage language;
     private PythonModule mainModule;
     private final PythonCore core;
+    private final List<ShutdownHook> shutdownHooks = new ArrayList<>();
     private final HashMap<Object, CallTarget> atExitHooks = new HashMap<>();
     private final HashMap<PythonNativeClass, CyclicAssumption> nativeClassStableAssumptions = new HashMap<>();
     private final AtomicLong globalId = new AtomicLong(Integer.MAX_VALUE * 2L + 4L);
@@ -516,6 +520,11 @@ public HashingStorage.Equivalence getSlowPathEquivalence() {
         return slowPathEquivalence;
     }
 
+    @TruffleBoundary
+    public void registerShutdownHook(ShutdownHook shutdownHook) {
+        shutdownHooks.add(shutdownHook);
+    }
+
     @TruffleBoundary
     public void registerShutdownHook(Object callable, CallTarget ct) {
         atExitHooks.put(callable, ct);
@@ -532,6 +541,9 @@ public void runShutdownHooks() {
         for (CallTarget f : atExitHooks.values()) {
             f.call();
         }
+        for (ShutdownHook h : shutdownHooks) {
+            h.call(this);
+        }
     }
 
     @TruffleBoundary

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/util/FileDeleteShutdownHook.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package com.oracle.graal.python.util;
+
+import java.io.IOException;
+
+import com.oracle.graal.python.runtime.PythonContext;
+import com.oracle.truffle.api.TruffleFile;
+
+public class FileDeleteShutdownHook implements ShutdownHook {
+    private final String path;
+
+    public FileDeleteShutdownHook(String path) {
+        this.path = path;
+    }
+
+    @Override
+    public void call(PythonContext context) {
+        TruffleFile truffleFile = context.getEnv().getTruffleFile(path);
+        try {
+            truffleFile.delete();
+        } catch (IOException ignored) {}
+    }
+}

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/util/ShutdownHook.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+package com.oracle.graal.python.util;
+
+import com.oracle.graal.python.runtime.PythonContext;
+
+public interface ShutdownHook {
+    void call(PythonContext context);
+}