JBR-9779 io-over-nio: RandomAccessFile now can open Windows pipes

The WinAPI function `GetFileAttributesW` actually opens a file and immediately closes it. If the file is a named pipe, it triggers a connection to the listener of the pipe. The listener closes the connection also asynchronously.

If the pipe is created with `nMaxInstances=1`, this code does not work:

```java
var path = Path.of("""\\.\pipe\openssh-ssh-agent""")

// `readAttributes` calls `GetFileAttributesW`, it opens the file and then closes it.
if (Files.readAttributes(path, BasicFileAttributes.class).isRegularFile()) {
  // Very probable race condition: the listener of the pipe has already received
  // the request of opening the file triggered by `GetFileAttributesW`
  // but hasn't closed the connection on their side yet.
  Files.newByteChannel(path);  // Fails with ERROR_PIPE_BUSY.
}
```

This revision adds an explicit check if some file looks like a Windows pipe. In these cases `RandomAccessFile` and `FileInputStream` don't call `Files.readAttributes` for the file.

Author: vladimirlagunov

src/java.base/share/classes/java/io/IoOverNioFileSystem.java

@@ -59,6 +59,7 @@
 import java.util.Arrays;
 import java.util.EnumSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Objects;
 import java.util.Set;
 
@@ -142,6 +143,22 @@ static Path getNioPath(File file, boolean mustBeRegularFile) {
             return nioPath;
         }
 
+        if (isWindowsPipe(nioPath)) {
+            // https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-createnamedpipea see nMaxInstances:
+            //
+            // Getting file attributes in this case is dangerous.
+            // GetFileAttributesW acquires a connection to the pipe internally,
+            // occupying a place on the server side.
+            // The server and the client are very likely two different processes, and it takes time to deliver
+            // the connection closing message to the server.
+            // If the caller invokes CreateFileW fast enough after GetFileAttributesW and nMaxInstances = 1,
+            // CreateFileW is called before the server closes the previous connection created by GetFileAttributesW
+            // and ERROR_PIPE_BUSY is returned.
+            //
+            // Anyway, `readAttributes(nioPath).isRegularFile()` returns true for pipes, so it's safe to return here.
+            return nioPath;
+        }
+
         // Two significant differences between the legacy java.io and java.nio.files:
         // * java.nio.file allows to open directories as streams, java.io.FileInputStream doesn't.
         // * java.nio.file doesn't work well with pseudo devices, i.e., `seek()` fails, while java.io works well.
@@ -158,6 +175,19 @@ static Path getNioPath(File file, boolean mustBeRegularFile) {
         return null;
     }
 
+    /**
+     * <a href="https://learn.microsoft.com/en-us/windows/win32/ipc/pipe-names">
+     * The pipe path format: {@code ^\\(\w+|\.)\pipe\.*}
+     * </a>
+     */
+    private static boolean isWindowsPipe(Path path) {
+        // A small JMH benchmark shows that this code takes less than a microsecond,
+        // and the JIT compiler does its job very well here.
+        return path.isAbsolute() &&
+                path.getRoot().toString().startsWith("\\\\") &&
+                path.getRoot().toString().toLowerCase(Locale.getDefault()).endsWith("\\pipe\\");
+    }
+
     private static boolean setPermission0(java.nio.file.FileSystem nioFs, File f, int access, boolean enable, boolean owneronly) {
         if (f.getPath().isEmpty()) {
             if (nioFs.getSeparator().equals("\\")) {

test/jdk/jb/java/io/IoOverNio/RandomAccessFileTest.java

@@ -24,11 +24,14 @@
 /* @test
  * @summary java.io.RandomAccessFileTest uses java.nio.file inside.
  * @library testNio
+ * @compile --enable-preview --source 27 RandomAccessFileTest.java
  * @run junit/othervm
  *      -Djava.nio.file.spi.DefaultFileSystemProvider=testNio.ManglingFileSystemProvider
  *      -Djbr.java.io.use.nio=true
  *      --add-opens jdk.unsupported/com.sun.nio.file=ALL-UNNAMED
  *      --add-opens java.base/java.io=ALL-UNNAMED
+ *      --enable-native-access=ALL-UNNAMED
+ *      --enable-preview
  *      RandomAccessFileTest
  */
 
@@ -45,6 +48,9 @@
 import java.io.EOFException;
 import java.io.File;
 import java.io.RandomAccessFile;
+import java.lang.foreign.*;
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.VarHandle;
 import java.lang.reflect.Field;
 import java.nio.channels.FileChannel;
 import java.nio.channels.FileLock;
@@ -55,6 +61,7 @@
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.Objects;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -293,4 +300,122 @@ public void testNoShareDelete() throws Exception {
             FileSystems.getDefault().provider().newFileChannel(file.toPath(), Collections.singleton(option)).close();
         }
     }
+
+    /**
+     * JBR-9779
+     */
+    @Test
+    public void testWindowsPipe() throws Throwable {
+        Assume.assumeTrue("Windows-only test", System.getProperty("os.name").toLowerCase().startsWith("win"));
+
+        // Creating a pipe.
+        Linker linker = Linker.nativeLinker();
+        SymbolLookup loader = SymbolLookup.libraryLookup("kernel32", Arena.global());
+
+        StructLayout captureLayout = Linker.Option.captureStateLayout();
+        VarHandle GetLastError = captureLayout.varHandle(MemoryLayout.PathElement.groupElement("GetLastError"));
+
+        MethodHandle CreateNamedPipeW = linker.downcallHandle(
+                loader.find("CreateNamedPipeW").get(),
+                FunctionDescriptor.of(ValueLayout.JAVA_LONG,
+                        ValueLayout.ADDRESS, ValueLayout.JAVA_INT, ValueLayout.JAVA_INT,
+                        ValueLayout.JAVA_INT, ValueLayout.JAVA_INT, ValueLayout.JAVA_INT, ValueLayout.JAVA_INT,
+                        ValueLayout.ADDRESS),
+                Linker.Option.captureCallState("GetLastError")
+        );
+
+        MethodHandle ConnectNamedPipe = linker.downcallHandle(
+                loader.find("ConnectNamedPipe").get(),
+                FunctionDescriptor.of(ValueLayout.JAVA_INT, ValueLayout.JAVA_LONG, ValueLayout.ADDRESS),
+                Linker.Option.captureCallState("GetLastError")
+        );
+
+        MethodHandle DisconnectNamedPipe = linker.downcallHandle(
+                loader.find("DisconnectNamedPipe").get(),
+                FunctionDescriptor.of(ValueLayout.JAVA_INT, ValueLayout.JAVA_LONG)
+        );
+
+        MethodHandle PeekNamedPipe = linker.downcallHandle(
+                loader.find("PeekNamedPipe").get(),
+                FunctionDescriptor.of(ValueLayout.JAVA_INT, ValueLayout.JAVA_LONG,
+                        ValueLayout.ADDRESS, ValueLayout.JAVA_INT, ValueLayout.ADDRESS,
+                        ValueLayout.ADDRESS, ValueLayout.ADDRESS)
+        );
+
+        MethodHandle CloseHandle = linker.downcallHandle(
+                loader.find("CloseHandle").get(),
+                FunctionDescriptor.of(ValueLayout.JAVA_INT, ValueLayout.JAVA_LONG)
+        );
+
+        String pipeName = "\\\\.\\pipe\\jbr-test-pipe-" + System.nanoTime();
+        Arena arena = Arena.ofAuto();
+        char[] nameChars = (pipeName + "\0").toCharArray();  // `char` on Windows is UTF-16, as WinAPI expects.
+        MemorySegment pipeWinPath = arena.allocateFrom(ValueLayout.JAVA_CHAR, nameChars);
+        MemorySegment capturedState = arena.allocate(captureLayout);
+
+        final long INVALID_HANDLE_VALUE = -1L;
+
+        long hPipe = (long) CreateNamedPipeW.invokeExact(
+                capturedState,
+                pipeWinPath,
+                0x00000003, // dwOpenMode = PIPE_ACCESS_DUPLEX
+                0x00000000, // dwPipeMode = PIPE_TYPE_BYTE
+                1, // nMaxInstances. Limit to 1 to force ERROR_PIPE_BUSY.
+                1024, // nOutBufferSize
+                1024, // nInBufferSize
+                0, // nDefaultTimeOut
+                MemorySegment.NULL // lpSecurityAttributes
+        );
+
+        if (hPipe == INVALID_HANDLE_VALUE) {
+            int errorCode = (int) GetLastError.get(capturedState);
+            throw new Exception("CreateNamedPipeW failed: " + errorCode);
+        }
+
+        AtomicBoolean keepRunning = new AtomicBoolean(true);
+        Thread serverThread = new Thread(() -> {
+            // This server accepts a connection and does nothing until the client disconnects explicitly.
+            try {
+                int i = 0;
+                while (keepRunning.get()) {
+                    int connected = (int) ConnectNamedPipe.invokeExact(capturedState, hPipe, MemorySegment.NULL);
+                    if (connected == 0) {
+                        int errorCode = (int) GetLastError.get(capturedState);
+                        // https://learn.microsoft.com/en-us/windows/win32/debug/system-error-codes--0-499-
+                        if (errorCode == 6 && !keepRunning.get()) { // ERROR_INVALID_HANDLE
+                            break;
+                        }
+                        throw new Exception("ConnectNamedPipe failed: " + errorCode);
+                    }
+                    try {
+                        int peekResult;
+                        do {
+                            // Random timeout. The timeout must be big enough to reveal possible consequent
+                            // attempts to connect to the pipe.
+                            Thread.sleep(1000);
+
+                            // Check if the pipe is still connected by peeking at it.
+                            peekResult = (int) PeekNamedPipe.invokeExact(hPipe, MemorySegment.NULL, 0,
+                                    MemorySegment.NULL, MemorySegment.NULL, MemorySegment.NULL);
+                        }
+                        while (keepRunning.get() && peekResult != 0);
+                    } finally {
+                        int disconnected = (int) DisconnectNamedPipe.invokeExact(hPipe);
+                    }
+                }
+            } catch (Throwable e) {
+                e.printStackTrace();
+            }
+        });
+
+        serverThread.setDaemon(true);
+        serverThread.start();
+
+        try {
+            new RandomAccessFile(pipeName, "rw").close();
+        } finally {
+            keepRunning.set(false);
+            int closed = (int) CloseHandle.invokeExact(hPipe);
+        }
+    }
 }