[feature] Improve EXPath File Module for W3C XQTS compliance

Add missing function overloads:
- file:read-text/read-text-lines 3-arg $fallback form
- file:create-temp-dir/create-temp-file 2-arg form

Fix error codes per EXPath File 4.0 spec:
- file:copy/move raise file:no-dir when target parent missing
- file:create-dir raises file:exists when path component is a file
- file:read-binary rejects negative $length with file:out-of-range
- file:write-binary validates $offset against file size

Fix readBinary hang: replace BinaryValueFromInputStream (which uses
CachingFilterInputStream/FilterInputStreamCacheMonitor infrastructure
that prevents clean BrokerPool shutdown) with BinaryValueFromBinaryString.
Reads file into byte[], base64-encodes, wraps in lightweight value type
with no open handles and no-op close(). Tradeoff: ~2.4x memory for file
content, acceptable for typical file module use cases.

Resolve relative paths against XQuery static base URI when set as a
file: URI, falling back to JVM working directory.

Detect XML-illegal characters in read-text/read-text-lines: raise
file:io-error by default, or replace with U+FFFD when $fallback=true.

QT4 XQTS expath-file: 183/190 (96.3%), 0 hangs, 0 errors.

Author: joewiz

extensions/expath/src/main/java/org/expath/exist/file/ExpathFileModule.java

@@ -50,6 +50,7 @@ public class ExpathFileModule extends AbstractInternalModule {
             new FunctionDef(FileProperties.signatures[6], FileProperties.class),
 
             // FileIO: read-text(1), read-text(2), read-text-lines(1), read-text-lines(2),
+            //         read-text(3-fallback), read-text-lines(3-fallback),
             //         read-binary(1), read-binary(2), read-binary(3)
             new FunctionDef(FileIO.signatures[0], FileIO.class),
             new FunctionDef(FileIO.signatures[1], FileIO.class),
@@ -58,6 +59,8 @@ public class ExpathFileModule extends AbstractInternalModule {
             new FunctionDef(FileIO.signatures[4], FileIO.class),
             new FunctionDef(FileIO.signatures[5], FileIO.class),
             new FunctionDef(FileIO.signatures[6], FileIO.class),
+            new FunctionDef(FileIO.signatures[7], FileIO.class),
+            new FunctionDef(FileIO.signatures[8], FileIO.class),
 
             // FileWrite: write(2), write(3), write-text(2), write-text(3),
             //            write-text-lines(2), write-text-lines(3), write-binary(2), write-binary(3)
@@ -81,7 +84,9 @@ public class ExpathFileModule extends AbstractInternalModule {
             new FunctionDef(FileAppend.signatures[6], FileAppend.class),
 
             // FileManipulation: copy, move, delete(1), delete(2), create-dir,
-            //                   create-temp-dir, create-temp-file, list(1), list(2), list(3),
+            //                   create-temp-dir(2), create-temp-dir(3),
+            //                   create-temp-file(2), create-temp-file(3),
+            //                   list(1), list(2), list(3),
             //                   children, descendants, list-roots
             new FunctionDef(FileManipulation.signatures[0], FileManipulation.class),
             new FunctionDef(FileManipulation.signatures[1], FileManipulation.class),
@@ -96,6 +101,8 @@ public class ExpathFileModule extends AbstractInternalModule {
             new FunctionDef(FileManipulation.signatures[10], FileManipulation.class),
             new FunctionDef(FileManipulation.signatures[11], FileManipulation.class),
             new FunctionDef(FileManipulation.signatures[12], FileManipulation.class),
+            new FunctionDef(FileManipulation.signatures[13], FileManipulation.class),
+            new FunctionDef(FileManipulation.signatures[14], FileManipulation.class),
 
             // FilePaths: name, parent, path-to-native, path-to-uri, resolve-path(1), resolve-path(2)
             new FunctionDef(FilePaths.signatures[0], FilePaths.class),

extensions/expath/src/main/java/org/expath/exist/file/ExpathFileModuleHelper.java

@@ -58,19 +58,57 @@ public static void checkDbaRole(final XQueryContext context, final Expression ex
 
     /**
      * Resolve a path string (file: URI or native path) to a {@link Path}.
+     * Relative paths are resolved against the JVM working directory.
      *
      * @param path the path string or file: URI
      * @param expression the calling expression (for error reporting)
      * @return the resolved Path
      * @throws XPathException if the path is invalid
      */
     public static Path getPath(final String path, final Expression expression) throws XPathException {
+        return getPath(path, expression, null);
+    }
+
+    /**
+     * Resolve a path string (file: URI or native path) to a {@link Path}.
+     * Relative paths are resolved against the XQuery static base URI if it is a
+     * file: URI, otherwise against the JVM working directory.
+     *
+     * @param path the path string or file: URI
+     * @param expression the calling expression (for error reporting)
+     * @param context the XQuery context (may be null)
+     * @return the resolved Path
+     * @throws XPathException if the path is invalid
+     */
+    public static Path getPath(final String path, final Expression expression, final XQueryContext context) throws XPathException {
         try {
             if (path.startsWith("file:")) {
                 return Paths.get(new URI(path));
-            } else {
-                return Paths.get(path);
             }
+
+            final Path p = Paths.get(path);
+            if (p.isAbsolute()) {
+                return p;
+            }
+
+            // Resolve relative paths against static base URI if available
+            if (context != null) {
+                try {
+                    final String baseUri = context.getBaseURI().getStringValue();
+                    if (baseUri != null && baseUri.startsWith("file:")) {
+                        final Path basePath = Paths.get(new URI(baseUri));
+                        // Base URI may point to a file; resolve against its parent directory
+                        final Path baseDir = java.nio.file.Files.isDirectory(basePath) ? basePath : basePath.getParent();
+                        if (baseDir != null) {
+                            return baseDir.resolve(p);
+                        }
+                    }
+                } catch (final Exception ignored) {
+                    // Fall through to default resolution
+                }
+            }
+
+            return p;
         } catch (final InvalidPathException e) {
             throw new XPathException(expression, ExpathFileErrorCode.INVALID_PATH,
                     "Invalid path: " + path + " - " + e.getMessage());

extensions/expath/src/main/java/org/expath/exist/file/FileAppend.java

@@ -138,7 +138,7 @@ public Sequence eval(final Sequence[] args, final Sequence contextSequence) thro
         ExpathFileModuleHelper.checkDbaRole(context, this);
 
         final String pathStr = args[0].getStringValue();
-        final Path path = ExpathFileModuleHelper.getPath(pathStr, this);
+        final Path path = ExpathFileModuleHelper.getPath(pathStr, this, context);
 
         checkParentDir(path);
 

extensions/expath/src/main/java/org/expath/exist/file/FileIO.java

@@ -36,7 +36,7 @@
 import org.exist.xquery.XPathException;
 import org.exist.xquery.XQueryContext;
 import org.exist.xquery.value.Base64BinaryValueType;
-import org.exist.xquery.value.BinaryValueFromInputStream;
+import org.exist.xquery.value.BinaryValueFromBinaryString;
 import org.exist.xquery.value.FunctionParameterSequenceType;
 import org.exist.xquery.value.FunctionReturnSequenceType;
 import org.exist.xquery.value.Sequence;
@@ -88,6 +88,24 @@ public class FileIO extends BasicFunction {
                     new SequenceType[]{FILE_PARAM, ENCODING_PARAM},
                     new FunctionReturnSequenceType(Type.STRING, Cardinality.ZERO_OR_MORE, "the lines of the file.")
             ),
+            // file:read-text($file as xs:string, $encoding as xs:string?, $fallback as xs:boolean) as xs:string
+            new FunctionSignature(
+                    new QName("read-text", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX),
+                    "Reads the contents of a file as text. If $fallback is true, invalid characters are replaced.",
+                    new SequenceType[]{FILE_PARAM, ENCODING_PARAM,
+                            new FunctionParameterSequenceType("fallback", Type.BOOLEAN, Cardinality.EXACTLY_ONE,
+                                    "If true, replace invalid characters with the Unicode replacement character.")},
+                    new FunctionReturnSequenceType(Type.STRING, Cardinality.EXACTLY_ONE, "the file contents as string.")
+            ),
+            // file:read-text-lines($file as xs:string, $encoding as xs:string?, $fallback as xs:boolean) as xs:string*
+            new FunctionSignature(
+                    new QName("read-text-lines", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX),
+                    "Reads the contents of a file as a sequence of lines. If $fallback is true, invalid characters are replaced.",
+                    new SequenceType[]{FILE_PARAM, ENCODING_PARAM,
+                            new FunctionParameterSequenceType("fallback", Type.BOOLEAN, Cardinality.EXACTLY_ONE,
+                                    "If true, replace invalid characters with the Unicode replacement character.")},
+                    new FunctionReturnSequenceType(Type.STRING, Cardinality.ZERO_OR_MORE, "the lines of the file.")
+            ),
             // file:read-binary($file as xs:string) as xs:base64Binary
             new FunctionSignature(
                     new QName("read-binary", ExpathFileModule.NAMESPACE_URI, ExpathFileModule.PREFIX),
@@ -130,7 +148,7 @@ public Sequence eval(final Sequence[] args, final Sequence contextSequence) thro
         ExpathFileModuleHelper.checkDbaRole(context, this);
 
         final String pathStr = args[0].getStringValue();
-        final Path path = ExpathFileModuleHelper.getPath(pathStr, this);
+        final Path path = ExpathFileModuleHelper.getPath(pathStr, this, context);
 
         if (!Files.exists(path)) {
             throw new XPathException(this, ExpathFileErrorCode.NOT_FOUND,
@@ -154,20 +172,27 @@ public Sequence eval(final Sequence[] args, final Sequence contextSequence) thro
 
     private Sequence readText(final Path path, final Sequence[] args) throws XPathException {
         final Charset encoding = getEncoding(args, 1);
+        final boolean fallback = args.length > 2 && !args[2].isEmpty()
+                && args[2].itemAt(0).toJavaObject(Boolean.class);
         try {
-            final String content = Files.readString(path, encoding);
+            final String content = readFileText(path, encoding, fallback);
             // Normalize newlines per spec: CR or CRLF -> LF
             final String normalized = content.replace("\r\n", "\n").replace("\r", "\n");
             return new StringValue(this, normalized);
+        } catch (final java.nio.charset.MalformedInputException e) {
+            throw new XPathException(this, ExpathFileErrorCode.IO_ERROR,
+                    "Invalid characters in file for encoding " + encoding.name());
         } catch (final IOException e) {
             throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage());
         }
     }
 
     private Sequence readTextLines(final Path path, final Sequence[] args) throws XPathException {
         final Charset encoding = getEncoding(args, 1);
+        final boolean fallback = args.length > 2 && !args[2].isEmpty()
+                && args[2].itemAt(0).toJavaObject(Boolean.class);
         try {
-            final String content = Files.readString(path, encoding);
+            final String content = readFileText(path, encoding, fallback);
             // Split at newline boundaries per spec
             final String[] lines = content.split("\r\n|\r|\n", -1);
             final ValueSequence result = new ValueSequence(lines.length);
@@ -177,48 +202,116 @@ private Sequence readTextLines(final Path path, final Sequence[] args) throws XP
                 result.add(new StringValue(this, lines[i]));
             }
             return result;
+        } catch (final java.nio.charset.MalformedInputException e) {
+            throw new XPathException(this, ExpathFileErrorCode.IO_ERROR,
+                    "Invalid characters in file for encoding " + encoding.name());
         } catch (final IOException e) {
             throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage());
         }
     }
 
     private Sequence readBinary(final Path path, final Sequence[] args) throws XPathException {
         final long offset = args.length > 1 && !args[1].isEmpty() ? args[1].itemAt(0).toJavaObject(Long.class) : 0;
-        final long length = args.length > 2 && !args[2].isEmpty() ? args[2].itemAt(0).toJavaObject(Long.class) : -1;
+        final boolean hasLength = args.length > 2 && !args[2].isEmpty();
+        final long length = hasLength ? args[2].itemAt(0).toJavaObject(Long.class) : -1;
 
         try {
             final long fileSize = Files.size(path);
-            if (offset < 0 || offset > fileSize) {
-                throw new XPathException(this, ExpathFileErrorCode.OUT_OF_RANGE,
-                        "Offset " + offset + " is out of range for file of size " + fileSize);
-            }
-            if (length < -1) {
-                throw new XPathException(this, ExpathFileErrorCode.OUT_OF_RANGE,
-                        "Length must not be negative: " + length);
-            }
-            if (length >= 0 && offset + length > fileSize) {
-                throw new XPathException(this, ExpathFileErrorCode.OUT_OF_RANGE,
-                        "Offset + length exceeds file size: " + (offset + length) + " > " + fileSize);
-            }
+            validateBinaryRange(offset, length, hasLength, fileSize);
+
+            final byte[] data = readBinaryData(path, offset, hasLength, length, fileSize);
+            final String base64 = java.util.Base64.getEncoder().encodeToString(data);
+            return new BinaryValueFromBinaryString(this, new Base64BinaryValueType(), base64);
+        } catch (final IOException e) {
+            throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage());
+        }
+    }
 
-            if (offset == 0 && length < 0) {
-                // Read entire file
-                final InputStream is = Files.newInputStream(path);
-                return BinaryValueFromInputStream.getInstance(context, new Base64BinaryValueType(), is, this);
+    private void validateBinaryRange(final long offset, final long length, final boolean hasLength, final long fileSize) throws XPathException {
+        if (offset < 0 || offset > fileSize) {
+            throw new XPathException(this, ExpathFileErrorCode.OUT_OF_RANGE,
+                    "Offset " + offset + " is out of range for file of size " + fileSize);
+        }
+        if (hasLength && length < 0) {
+            throw new XPathException(this, ExpathFileErrorCode.OUT_OF_RANGE,
+                    "Length must not be negative: " + length);
+        }
+        if (hasLength && offset + length > fileSize) {
+            throw new XPathException(this, ExpathFileErrorCode.OUT_OF_RANGE,
+                    "Offset + length exceeds file size: " + (offset + length) + " > " + fileSize);
+        }
+    }
+
+    private byte[] readBinaryData(final Path path, final long offset, final boolean hasLength, final long length, final long fileSize) throws IOException {
+        if (offset == 0 && !hasLength) {
+            return Files.readAllBytes(path);
+        }
+        try (final RandomAccessFile raf = new RandomAccessFile(path.toFile(), "r")) {
+            raf.seek(offset);
+            final int readLen = hasLength ? (int) length : (int) (fileSize - offset);
+            final byte[] data = new byte[readLen];
+            raf.readFully(data);
+            return data;
+        }
+    }
+
+    /**
+     * Reads a file as text with the given encoding.
+     * If fallback is true, malformed byte sequences and XML-illegal characters
+     * are replaced with U+FFFD. Otherwise, an IOException is thrown if the file
+     * contains malformed bytes or XML-illegal characters.
+     */
+    private String readFileText(final Path path, final Charset encoding, final boolean fallback) throws IOException {
+        final String content;
+        if (fallback) {
+            final java.nio.charset.CharsetDecoder decoder = encoding.newDecoder()
+                    .onMalformedInput(java.nio.charset.CodingErrorAction.REPLACE)
+                    .onUnmappableCharacter(java.nio.charset.CodingErrorAction.REPLACE)
+                    .replaceWith("\uFFFD");
+            final byte[] bytes = Files.readAllBytes(path);
+            content = decoder.decode(java.nio.ByteBuffer.wrap(bytes)).toString();
+            // Replace XML-illegal characters with U+FFFD
+            return replaceXmlIllegalChars(content);
+        } else {
+            content = Files.readString(path, encoding);
+            // Check for XML-illegal characters
+            checkXmlIllegalChars(content);
+            return content;
+        }
+    }
+
+    /**
+     * Check if a string contains characters illegal in XML 1.0 and throw IOException if so.
+     * XML 1.0 allows: #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]
+     */
+    private void checkXmlIllegalChars(final String text) throws IOException {
+        for (int i = 0; i < text.length(); i++) {
+            final char c = text.charAt(i);
+            if (c < 0x20 && c != 0x9 && c != 0xA && c != 0xD) {
+                throw new IOException("File contains XML-illegal character U+" +
+                        String.format("%04X", (int) c) + " at position " + i);
+            }
+            if (c >= 0xFFFE) {
+                throw new IOException("File contains XML-illegal character U+" +
+                        String.format("%04X", (int) c) + " at position " + i);
             }
+        }
+    }
 
-            // Partial read
-            try (final RandomAccessFile raf = new RandomAccessFile(path.toFile(), "r")) {
-                raf.seek(offset);
-                final int readLen = length >= 0 ? (int) length : (int) (fileSize - offset);
-                final byte[] data = new byte[readLen];
-                raf.readFully(data);
-                final InputStream bis = new java.io.ByteArrayInputStream(data);
-                return BinaryValueFromInputStream.getInstance(context, new Base64BinaryValueType(), bis, this);
+    /**
+     * Replace characters illegal in XML 1.0 with U+FFFD.
+     */
+    private String replaceXmlIllegalChars(final String text) {
+        final StringBuilder sb = new StringBuilder(text.length());
+        for (int i = 0; i < text.length(); i++) {
+            final char c = text.charAt(i);
+            if ((c < 0x20 && c != 0x9 && c != 0xA && c != 0xD) || c >= 0xFFFE) {
+                sb.append('\uFFFD');
+            } else {
+                sb.append(c);
             }
-        } catch (final IOException e) {
-            throw new XPathException(this, ExpathFileErrorCode.IO_ERROR, e.getMessage());
         }
+        return sb.toString();
     }
 
     private Charset getEncoding(final Sequence[] args, final int index) throws XPathException {