Transparently add the \\?\ prefix to Win32 calls for extended length path handling (#1257)

On Windows, there is a built-in maximum path limitation of 260 characters under most conditions. This can be extended to 32767 characters under either of the following two conditions:

- Adding the longPathAware attribute to the executable's manifest AND enabling the LongPathsEnabled system-wide registry key or group policy.
- Ensuring fully qualified paths passed to Win32 APIs are prefixed with \?\

Unfortunately, the former is not realistic for the Swift ecosystem, since it requires developers to have awareness of this specific Windows limitation, AND set longPathAware in their apps' manifest AND expect end users of those apps to change their system configuration.

Instead, this patch transparently prefixes all eligible paths in calls to Win32 APIs with the \?\ prefix to allow them to work with paths longer than 260 characters without requiring the caller of Foundation to manually prefix the paths.

See https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation for more info.

Author: jakepetroules

Sources/FoundationEssentials/Data/Data+Writing.swift

@@ -171,7 +171,7 @@ private func createTemporaryFile(at destinationPath: String, inPath: PathOrURL,
             guard _mktemp_s(templateFileSystemRep, strlen(templateFileSystemRep) + 1) == 0 else {
                 throw CocoaError.errorWithFilePath(inPath, errno: errno, reading: false, variant: variant)
             }
-            let fd = String(cString: templateFileSystemRep).withCString(encodedAs: UTF16.self) {
+            let fd = try String(cString: templateFileSystemRep).withNTPathRepresentation {
                 openFileDescriptorProtected(path: $0, flags: _O_BINARY | _O_CREAT | _O_EXCL | _O_RDWR, options: options)
             }
 #else

Sources/FoundationEssentials/FileManager/FileManager+Directories.swift

@@ -257,36 +257,6 @@ extension _FileManagerImpl {
         try fileManager.createDirectory(atPath: path, withIntermediateDirectories: createIntermediates, attributes: attributes)
     }
 
-#if os(Windows)
-    /// If `path` is absolute, this is the same as `path.withNTPathRepresentation`.
-    /// If `path` is relative, this creates an absolute path of `path` relative to `currentDirectoryPath` and runs
-    /// `body` with that path.
-    private func withAbsoluteNTPathRepresentation<Result>(
-        of path: String,
-        _ body: (UnsafePointer<WCHAR>) throws -> Result
-    ) throws -> Result {
-        try path.withNTPathRepresentation { pwszPath in
-            if !PathIsRelativeW(pwszPath) {
-                // We already have an absolute path. Nothing to do
-                return try body(pwszPath)
-            }
-            guard let currentDirectoryPath else {
-                preconditionFailure("We should always have a current directory on Windows")
-            }
-
-            // We have a relateive path. Make it absolute.
-            let absoluteUrl = URL(
-                filePath: path,
-                directoryHint: .isDirectory,
-                relativeTo: URL(filePath: currentDirectoryPath, directoryHint: .isDirectory)
-            )
-            return try absoluteUrl.path.withNTPathRepresentation { pwszPath in
-                return try body(pwszPath)
-            }
-        }
-    }
-#endif
-    
     func createDirectory(
         atPath path: String,
         withIntermediateDirectories createIntermediates: Bool,
@@ -301,7 +271,7 @@ extension _FileManagerImpl {
         if createIntermediates {
             // `SHCreateDirectoryExW` requires an absolute path while `CreateDirectoryW` works based on the current working
             // directory.
-            try withAbsoluteNTPathRepresentation(of: path) { pwszPath in
+            try path.withNTPathRepresentation { pwszPath in
                 let errorCode = SHCreateDirectoryExW(nil, pwszPath, &saAttributes)
                 guard let errorCode = DWORD(exactly: errorCode) else {
                     // `SHCreateDirectoryExW` returns `Int` but all error codes are defined in terms of `DWORD`, aka

Sources/FoundationEssentials/FileManager/FileManager+SymbolicLinks.swift

@@ -64,7 +64,7 @@ extension _FileManagerImpl {
 
         try path.withNTPathRepresentation { lpSymlinkFileName in
             try destPath.withFileSystemRepresentation {
-                try String(cString: $0!).withCString(encodedAs: UTF16.self) { lpTargetFileName in
+                try String(cString: $0!).withNTPathRepresentation(relative: true) { lpTargetFileName in
                     if CreateSymbolicLinkW(lpSymlinkFileName, lpTargetFileName, SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE | (bIsDirectory ? SYMBOLIC_LINK_FLAG_DIRECTORY : 0)) == 0 {
                         throw CocoaError.errorWithFilePath(path, win32: GetLastError(), reading: false)
                     }

Sources/FoundationEssentials/FileManager/FileOperations.swift

@@ -829,7 +829,7 @@ enum _FileOperations {
         try src.withNTPathRepresentation { pwszSource in
             var faAttributes: WIN32_FILE_ATTRIBUTE_DATA = .init()
             guard GetFileAttributesExW(pwszSource, GetFileExInfoStandard, &faAttributes) else {
-                throw CocoaError.errorWithFilePath(.fileReadNoSuchFile, src, variant: bCopyFile ? "Copy" : "Link", source: src, destination: dst)
+                throw CocoaError.errorWithFilePath(src, win32: GetLastError(), reading: true, variant: bCopyFile ? "Copy" : "Link", source: src, destination: dst)
             }
 
             guard delegate.shouldPerformOnItemAtPath(src, to: dst) else { return }

Sources/FoundationEssentials/String/String+Internals.swift

@@ -23,7 +23,12 @@ import Darwin
 import WinSDK
 
 extension String {
-    package func withNTPathRepresentation<Result>(_ body: (UnsafePointer<WCHAR>) throws -> Result) throws -> Result {
+    /// Invokes `body` with a resolved and potentially `\\?\`-prefixed version of the pointee,
+    /// to ensure long paths greater than MAX_PATH (260) characters are handled correctly.
+    ///
+    /// - parameter relative: Returns the original path without transforming through GetFullPathNameW + PathCchCanonicalizeEx, if the path is relative.
+    /// - seealso: https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation
+    package func withNTPathRepresentation<Result>(relative: Bool = false, _ body: (UnsafePointer<WCHAR>) throws -> Result) throws -> Result {
         guard !isEmpty else {
             throw CocoaError.errorWithFilePath(.fileReadInvalidFileName, "")
         }
@@ -35,15 +40,42 @@ extension String {
         // leading slash indicates a rooted path on the drive for the current
         // working directory.
         return try Substring(self.utf8.dropFirst(bLeadingSlash ? 1 : 0)).withCString(encodedAs: UTF16.self) { pwszPath in
+            if relative && PathIsRelativeW(pwszPath) {
+                return try body(pwszPath)
+            }
+
             // 1. Normalize the path first.
+            // Contrary to the documentation, this works on long paths independently
+            // of the registry or process setting to enable long paths (but it will also
+            // not add the \\?\ prefix required by other functions under these conditions).
             let dwLength: DWORD = GetFullPathNameW(pwszPath, 0, nil, nil)
-            return try withUnsafeTemporaryAllocation(of: WCHAR.self, capacity: Int(dwLength)) {
-                guard GetFullPathNameW(pwszPath, DWORD($0.count), $0.baseAddress, nil) > 0 else {
+            return try withUnsafeTemporaryAllocation(of: WCHAR.self, capacity: Int(dwLength)) { pwszFullPath in
+                guard (1..<dwLength).contains(GetFullPathNameW(pwszPath, DWORD(pwszFullPath.count), pwszFullPath.baseAddress, nil)) else {
                     throw CocoaError.errorWithFilePath(self, win32: GetLastError(), reading: true)
                 }
 
-                // 2. Perform the operation on the normalized path.
-                return try body($0.baseAddress!)
+                // 1.5 Leave \\.\ prefixed paths alone since device paths are already an exact representation and PathCchCanonicalizeEx will mangle these.
+                if let base = pwszFullPath.baseAddress,
+                    base[0] == UInt16(UInt8._backslash),
+                    base[1] == UInt16(UInt8._backslash),
+                    base[2] == UInt16(UInt8._period),
+                    base[3] == UInt16(UInt8._backslash) {
+                    return try body(base)
+                }
+
+                // 2. Canonicalize the path.
+                // This will add the \\?\ prefix if needed based on the path's length.
+                var pwszCanonicalPath: LPWSTR?
+                let flags: ULONG = PATHCCH_ALLOW_LONG_PATHS
+                let result = PathAllocCanonicalize(pwszFullPath.baseAddress, flags, &pwszCanonicalPath)
+                if let pwszCanonicalPath {
+                    defer { LocalFree(pwszCanonicalPath) }
+                    if result == S_OK {
+                        // 3. Perform the operation on the normalized path.
+                        return try body(pwszCanonicalPath)
+                    }
+                }
+                throw CocoaError.errorWithFilePath(self, win32: WIN32_FROM_HRESULT(result), reading: true)
             }
         }
     }

Tests/FoundationEssentialsTests/FileManager/FileManagerTests.swift

@@ -1072,4 +1072,92 @@ final class FileManagerTests : XCTestCase {
             XCTAssertEqual($0.contents(atPath: "a\u{301}/test"), data)
         }
     }
+
+    /// Tests that Foundation can correctly handle "long paths" (paths of 260 to 32767 chacters long) on Windows.
+    func testWindowsLongPathSupport() throws {
+        #if !os(Windows)
+        throw XCTSkip("This test is not applicable for this platform")
+        #else
+        // Create a directory with the absolute maximum path _component_ length of 255;
+        // this will guarantee the full playground path is well over 260 characters.
+        // Throw some Unicode in there for good measure, since only wide-character APIs support it.
+        let dirName = String(repeating: UUID().uuidString, count: 7) + "你好！"
+        XCTAssertEqual(dirName.count, 255)
+        XCTAssertEqual(dirName.utf16.count, 255)
+
+        try FileManagerPlayground {
+            Directory(dirName) {
+            }
+        }.test {
+            // Call every function that can call into withNTPathRepresentation with an overlong path and ensure it succeeds.
+            let fileName = UUID().uuidString
+            let cwd = try XCTUnwrap($0.currentDirectoryPath)
+
+            XCTAssertTrue($0.createFile(atPath: dirName + "/" + fileName, contents: nil))
+
+            let dirURL = URL(filePath: dirName, directoryHint: .checkFileSystem)
+            XCTAssertTrue(dirURL.hasDirectoryPath)
+
+            let fileURL = URL(filePath: dirName + "/" + fileName, directoryHint: .checkFileSystem)
+            XCTAssertFalse(fileURL.hasDirectoryPath)
+
+            XCTAssertTrue($0.fileExists(atPath: dirName + "/" + fileName))
+            XCTAssertTrue($0.isReadableFile(atPath: dirName + "/" + fileName))
+            XCTAssertTrue($0.isWritableFile(atPath: dirName + "/" + fileName))
+
+            // SHGetFileInfoW is documented to be limited to MAX_PATH, but appears to support long paths anyways (or at least does for SHGFI_EXETYPE).
+            // https://learn.microsoft.com/en-us/windows/win32/api/shellapi/nf-shellapi-shgetfileinfow
+            XCTAssertNoThrow(try Data().write(to: URL(fileURLWithPath: dirName + "/" + fileName + ".bat")))
+            XCTAssertTrue($0.isExecutableFile(atPath: dirName + "/" + fileName + ".bat"))
+            XCTAssertFalse($0.isExecutableFile(atPath: dirName + "/" + fileName))
+
+            XCTAssertNoThrow(try $0.attributesOfItem(atPath: dirName + "/" + fileName))
+            XCTAssertNoThrow(try $0.setAttributes([.modificationDate: Date()], ofItemAtPath: dirName + "/" + fileName))
+            XCTAssertNoThrow(try $0.attributesOfFileSystem(forPath: dirName + "/" + fileName))
+
+            XCTAssertNoThrow(try Data(contentsOf: URL(fileURLWithPath: dirName + "/" + fileName)))
+
+            XCTAssertNoThrow(try Data("hello".utf8).write(to: URL(fileURLWithPath: dirName + "/" + fileName)))
+            XCTAssertNoThrow(try Data("hello".utf8).write(to: URL(fileURLWithPath: dirName + "/" + fileName), options: .atomic))
+
+            XCTAssertNoThrow(try Data("hello".utf8).write(to: URL(fileURLWithPath: dirName + "/" + fileName + ".v2")))
+            XCTAssertTrue($0.contentsEqual(atPath: dirName + "/" + fileName, andPath: dirName + "/" + fileName + ".v2"))
+
+            XCTAssertEqual(try $0.subpathsOfDirectory(atPath: dirName).sorted(), [
+                fileName,
+                fileName + ".bat",
+                fileName + ".v2"
+            ])
+
+            XCTAssertNoThrow(try $0.createDirectory(at: URL(fileURLWithPath: dirName + "/" + "subdir1"), withIntermediateDirectories: false))
+
+            // SHCreateDirectoryExW's path argument is limited to 248 characters, and the \\?\ prefix doesn't help.
+            // https://learn.microsoft.com/en-us/windows/win32/api/shlobj_core/nf-shlobj_core-shcreatedirectoryexw
+            XCTAssertThrowsError(try $0.createDirectory(at: URL(fileURLWithPath: dirName + "/" + "subdir2" + "/" + "subdir3"), withIntermediateDirectories: true))
+
+            // SetCurrentDirectory seems to be limited to MAX_PATH unconditionally, counter to the documentation.
+            // https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-setcurrentdirectory
+            // https://github.com/MicrosoftDocs/feedback/issues/1441
+            XCTAssertFalse($0.changeCurrentDirectoryPath(dirName + "/" + "subdir1"))
+
+            XCTAssertNoThrow(try $0.createSymbolicLink(atPath: dirName + "/" + "lnk", withDestinationPath: fileName))
+            XCTAssertNoThrow(try $0.createSymbolicLink(atPath: dirName + "/" + "lnk2", withDestinationPath: cwd + "/" + dirName + "/" + fileName))
+            XCTAssertEqual(try $0.destinationOfSymbolicLink(atPath: dirName + "/" + "lnk"), fileName)
+            XCTAssertEqual(try $0.destinationOfSymbolicLink(atPath: dirName + "/" + "lnk2"), cwd + "\\" + dirName + "\\" + fileName)
+
+            XCTAssertEqual((cwd + "/" + dirName + "/" + "lnk").resolvingSymlinksInPath, (cwd + "/" + dirName + "/" + fileName).resolvingSymlinksInPath)
+
+            XCTAssertNoThrow(try $0.createDirectory(at: URL(fileURLWithPath: dirName + "/" + "subdir2"), withIntermediateDirectories: false))
+            XCTAssertNoThrow(try $0.createDirectory(at: URL(fileURLWithPath: dirName + "/" + "subdir2" + "/" + "subdir3"), withIntermediateDirectories: false))
+            XCTAssertNoThrow(try Data().write(to: URL(fileURLWithPath: dirName + "/" + "subdir2" + "/" + "subdir3" + "/" + "somefile")))
+            XCTAssertNoThrow(try Data().write(to: URL(fileURLWithPath: dirName + "/" + "subdir2" + "/" + "subdir3" + "/" + "somefile2")))
+            XCTAssertNoThrow(try $0.moveItem(atPath: dirName + "/" + "subdir2" + "/" + "subdir3" + "/" + "somefile2", toPath: dirName + "/" + "subdir2" + "/" + "subdir3" + "/" + "somefile3"))
+            XCTAssertNoThrow(try $0.moveItem(atPath: dirName + "/" + "subdir2" + "/" + "subdir3", toPath: dirName + "/" + "subdir2" + "/" + "subdir3.delete"))
+            XCTAssertNoThrow(try $0.linkItem(atPath: dirName + "/" + "subdir2" + "/" + "subdir3.delete", toPath: dirName + "/" + "subdir2" + "/" + "subdir3.delete.lnk"))
+            XCTAssertNoThrow(try $0.linkItem(atPath: dirName + "/" + "subdir2", toPath: dirName + "/" + "subdir2.lnk"))
+            XCTAssertNoThrow(try $0.removeItem(atPath: dirName + "/" + "subdir2" + "/" + "subdir3.delete" + "/" + "somefile3"))
+            XCTAssertNoThrow(try $0.removeItem(atPath: dirName + "/" + "subdir2"))
+        }
+        #endif
+    }
 }