[bugfix] Improve handling of XML BOM

Closes eXist-db/exist#5610

Author: adamretter

exist-core/pom.xml

@@ -677,10 +677,13 @@
                             <header>${project.parent.relativePath}/../elemental-parent/elemental-LGPL-21-ONLY-license.template.txt</header>
                             <includes>
                                 <include>project-suppression.xml</include>
+                                <include>src/main/java/org/exist/util/ByteOrderMark.java</include>
                                 <include>src/test/java/org/exist/xquery/ImportFromPkgTest.java</include>
                                 <include>src/test/java/org/exist/xquery/WatchdogTest.java</include>
                                 <include>src/test/java/org/exist/xquery/value/DateTimeTypesTest.java</include>
                                 <include>src/test/java/org/exist/xquery/functions/fn/FunXmlToJsonTest.java</include>
+                                <include>src/test/java/org/exist/xquery/functions/fn/ParsingFunctionsTest.java</include>
+                                <include>src/test/java/org/exist/xquery/functions/xmldb/XMLDBStoreTest.java</include>
                                 <include>src/test/resources-filtered/org/exist/xquery/import-from-pkg-test.conf.xml</include>
                             </includes>
                         </licenseSet>
@@ -767,8 +770,10 @@
                                 <include>src/main/java/org/exist/xmldb/RemoteRestoreService.java</include>
                                 <include>src/main/java/org/exist/xmlrpc/ExistRpcTypeFactory.java</include>
                                 <include>src/main/java/org/exist/xquery/functions/fn/FunUriCollection.java</include>
+                                <include>src/main/java/org/exist/xquery/functions/fn/ParsingFunctions.java</include>
                                 <include>src/main/java/org/exist/xquery/functions/system/GetUptime.java</include>
                                 <include>src/main/java/org/exist/xquery/functions/system/Shutdown.java</include>
+                                <include>src/main/java/org/exist/xquery/functions/xmldb/XMLDBStore.java</include>
                                 <include>src/main/java/org/exist/xquery/value/AbstractDateTimeValue.java</include>
                                 <include>src/main/java/org/exist/xquery/value/Type.java</include>
                                 <include>src/main/java/org/exist/xslt/EXistURIResolver.java</include>
@@ -886,6 +891,7 @@
                                 <exclude>src/main/java/org/exist/test/runner/XMLTestRunner.java</exclude>
                                 <exclude>src/main/java/org/exist/test/runner/XQueryTestRunner.java</exclude>
                                 <exclude>src/main/java/org/exist/test/runner/XSuite.java</exclude>
+                                <exclude>src/main/java/org/exist/util/ByteOrderMark.java</exclude>
                                 <exclude>src/main/java/org/exist/util/Collations.java</exclude>
                                 <exclude>src/main/java/org/exist/util/crypto/digest/DigestType.java</exclude>
                                 <exclude>src/main/java/org/exist/webstart/JnlpJarFiles.java</exclude>
@@ -897,8 +903,12 @@
                                 <exclude>src/main/java/org/exist/xquery/functions/fn/FunXmlToJson.java</exclude>
                                 <exclude>src/test/java/org/exist/xquery/functions/fn/FunXmlToJsonTest.java</exclude>
                                 <exclude>src/main/java/org/exist/xquery/functions/fn/LoadXQueryModule.java</exclude>
+                                <exclude>src/main/java/org/exist/xquery/functions/fn/ParsingFunctions.java</exclude>
+                                <exclude>src/test/java/org/exist/xquery/functions/fn/ParsingFunctionsTest.java</exclude>
                                 <exclude>src/main/java/org/exist/xquery/functions/system/GetUptime.java</exclude>
                                 <exclude>src/main/java/org/exist/xquery/functions/system/Shutdown.java</exclude>
+                                <exclude>src/main/java/org/exist/xquery/functions/xmldb/XMLDBStore.java</exclude>
+                                <exclude>src/test/java/org/exist/xquery/functions/xmldb/XMLDBStoreTest.java</exclude>
                                 <exclude>src/main/java/org/exist/xquery/value/AbstractDateTimeValue.java</exclude>
                                 <exclude>src/test/java/org/exist/xquery/value/DateTimeTypesTest.java</exclude>
                                 <exclude>src/main/java/org/exist/xquery/value/Type.java</exclude>

exist-core/src/main/java/org/exist/util/ByteOrderMark.java

@@ -0,0 +1,54 @@
+/*
+ * Elemental
+ * Copyright (C) 2024, Evolved Binary Ltd
+ *
+ * [email protected]
+ * https://www.evolvedbinary.com | https://www.elemental.xyz
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; version 2.1.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+package org.exist.util;
+
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Byte Order Mark utilities.
+ *
+ * @author <a href="mailto:[email protected]">Adam Retter</a>
+ */
+public class ByteOrderMark {
+    public static final byte[] UTF8_BOM_BYTES = new byte[] {(byte)0xEF, (byte)0xBB, (byte)0xBF};
+    public static final String UTF8_BOM = new String(UTF8_BOM_BYTES, StandardCharsets.UTF_8);
+
+
+    public static final byte[] UTF16_BE_BOM_BYTES = new byte[] {(byte)0xFE, (byte)0xFF};
+    public static final String UTF16_BE_BOM = new String(UTF16_BE_BOM_BYTES, StandardCharsets.UTF_16BE);
+
+    public static final byte[] UTF16_LE_BOM_BYTES = new byte[] {(byte)0xFF, (byte)0xFE};
+    public static final String UTF16_LE_BOM = new String(UTF16_LE_BOM_BYTES, StandardCharsets.UTF_16LE);
+
+    /**
+     * Strip BOM from the start of an XML string.
+     *
+     * @param xml the XML as a string
+     *
+     * @return the XML without a BOM.
+     */
+    public static String stripXmlBom(final String xml) {
+        if (xml.startsWith(UTF8_BOM) || xml.startsWith(UTF16_BE_BOM) || xml.startsWith(UTF16_LE_BOM)) {
+            return xml.substring(1);
+        }
+        return xml;
+    }
+}

exist-core/src/main/java/org/exist/xquery/functions/fn/ParsingFunctions.java

@@ -1,4 +1,28 @@
 /*
+ * Elemental
+ * Copyright (C) 2024, Evolved Binary Ltd
+ *
+ * [email protected]
+ * https://www.evolvedbinary.com | https://www.elemental.xyz
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; version 2.1.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * NOTE: Parts of this file contain code from 'The eXist-db Authors'.
+ *       The original license header is included below.
+ *
+ * =====================================================================
+ *
  * eXist-db Open Source Native XML Database
  * Copyright (C) 2001 The eXist-db Authors
  *
@@ -40,6 +64,9 @@
 
 import java.io.IOException;
 import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
+
+import static org.exist.util.ByteOrderMark.stripXmlBom;
 
 public class ParsingFunctions extends BasicFunction {
 
@@ -103,7 +130,8 @@ private Sequence parse(final String xmlContent, final Sequence[] args) throws XP
         }
     }
 
-    private ValidationReport validate(final String xmlContent, final SAXAdapter saxAdapter) throws XPathException {
+    private ValidationReport validate(String xmlContent, final SAXAdapter saxAdapter) throws XPathException {
+        xmlContent = stripXmlBom(xmlContent);
         final String xml;
 	    if (isCalledAs("parse-xml-fragment")) {
             xml = "<" + FRAGMENT_WRAPPER_NAME + ">" + xmlContent + "</" + FRAGMENT_WRAPPER_NAME + ">";

exist-core/src/main/java/org/exist/xquery/functions/xmldb/XMLDBStore.java

@@ -1,4 +1,28 @@
 /*
+ * Elemental
+ * Copyright (C) 2024, Evolved Binary Ltd
+ *
+ * [email protected]
+ * https://www.evolvedbinary.com | https://www.elemental.xyz
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; version 2.1.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * NOTE: Parts of this file contain code from 'The eXist-db Authors'.
+ *       The original license header is included below.
+ *
+ * =====================================================================
+ *
  * eXist-db Open Source Native XML Database
  * Copyright (C) 2001 The eXist-db Authors
  *
@@ -65,12 +89,14 @@
 import org.xmldb.api.modules.BinaryResource;
 import org.xmldb.api.modules.XMLResource;
 
+import static org.exist.util.ByteOrderMark.stripXmlBom;
 import static org.exist.xquery.FunctionDSL.*;
 import static org.exist.xquery.XPathException.execAndAddErrorIfMissing;
 import static org.exist.xquery.functions.xmldb.XMLDBModule.functionSignature;
 import static org.exist.xquery.functions.xmldb.XMLDBModule.functionSignatures;
 
 /**
+ * @author <a href="mailto:[email protected]">Adam Retter</a>
  * @author wolf
  */
 public class XMLDBStore extends XMLDBAbstractCollectionManipulator {
@@ -196,7 +222,7 @@ public Sequence evalWithCollection(Collection collection, Sequence[] args, Seque
                 }
 
                 if (Type.subTypeOf(item.getType(), Type.STRING)) {
-                    resource.setContent(item.getStringValue());
+                    resource.setContent(stripXmlBom(item.getStringValue()));
 
                 } else if (item.getType() == Type.BASE64_BINARY) {
                     resource.setContent(((BinaryValue) item).toJavaObject());

exist-core/src/test/java/org/exist/xquery/functions/fn/ParsingFunctionsTest.java

@@ -0,0 +1,112 @@
+/*
+ * Elemental
+ * Copyright (C) 2024, Evolved Binary Ltd
+ *
+ * [email protected]
+ * https://www.evolvedbinary.com | https://www.elemental.xyz
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; version 2.1.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+package org.exist.xquery.functions.fn;
+
+import org.exist.test.ExistXmldbEmbeddedServer;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.xmldb.api.base.XMLDBException;
+
+import java.nio.charset.StandardCharsets;
+
+import static org.exist.util.ByteOrderMark.*;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author <a href="mailto:[email protected]">Adam Retter</a>
+ */
+public class ParsingFunctionsTest {
+
+    private static final String UTF8_DECL = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>";
+
+    @ClassRule
+    public static final ExistXmldbEmbeddedServer existEmbeddedServer = new ExistXmldbEmbeddedServer(false, true, true);
+
+    @Test
+    public void parseWithoutBomWithoutDecl() throws XMLDBException {
+        final String query = "parse-xml('<elem1/>')";
+        final String result = existEmbeddedServer.executeOneValue(query);
+        assertEquals("<elem1/>", result);
+    }
+
+    @Test
+    public void parseWithoutBomWithDecl() throws XMLDBException {
+        final String query = "parse-xml('" + UTF8_DECL + "<elem2/>')";
+        final String result = existEmbeddedServer.executeOneValue(query);
+        assertEquals("<elem2/>", result);
+    }
+
+    @Test
+    public void parseWithUtf8BomWithoutDecl() throws XMLDBException {
+        final String query = "parse-xml('" + UTF8_BOM + "<elem3/>')";
+        final String result = existEmbeddedServer.executeOneValue(query);
+        assertEquals("<elem3/>", result);
+    }
+
+    @Test
+    public void parseWithUtf8BomWithDecl() throws XMLDBException {
+        final String query = "parse-xml('" + UTF8_BOM + UTF8_DECL + "<elem4/>')";
+        final String result = existEmbeddedServer.executeOneValue(query);
+        assertEquals("<elem4/>", result);
+    }
+
+    @Test
+    public void parseWithUtf16BEBomWithoutDecl() throws XMLDBException {
+        final String query = "parse-xml('" + UTF16_BE_BOM + new String(new byte[]{0x00, 0x3c, 0x00, 0x65, 0x00, 0x6c, 0x00, 0x65, 0x00, 0x6d, 0x00, 0x35, 0x00, 0x2f, 0x00, 0x3e}, StandardCharsets.UTF_16BE) + "')";
+        final String result = existEmbeddedServer.executeOneValue(query);
+        assertEquals("<elem5/>", result);
+    }
+
+    @Test
+    public void parseWithUtf16LEBomWithoutDecl() throws XMLDBException {
+        final String query = "parse-xml('" + UTF16_LE_BOM + new String(new byte[]{0x3c, 0x00, 0x65, 0x00, 0x6c, 0x00, 0x65, 0x00, 0x6d, 0x00, 0x36, 0x00, 0x2f, 0x00, 0x3e, 0x00}, StandardCharsets.UTF_16LE) + "')";
+        final String result = existEmbeddedServer.executeOneValue(query);
+        assertEquals("<elem6/>", result);
+    }
+
+    @Test
+    public void parseFragmentWithoutBomWithoutDecl() throws XMLDBException {
+        final String query = "parse-xml-fragment('<elem1/>')";
+        final String result = existEmbeddedServer.executeOneValue(query);
+        assertEquals("<elem1/>", result);
+    }
+
+    @Test
+    public void parseFragmentWithUtf8BomWithoutDecl() throws XMLDBException {
+        final String query = "parse-xml-fragment('" + UTF8_BOM + "<elem3/>')";
+        final String result = existEmbeddedServer.executeOneValue(query);
+        assertEquals("<elem3/>", result);
+    }
+
+    @Test
+    public void parseFragmentWithUtf16BEBomWithoutDecl() throws XMLDBException {
+        final String query = "parse-xml-fragment('" + UTF16_BE_BOM + new String(new byte[]{0x00, 0x3c, 0x00, 0x65, 0x00, 0x6c, 0x00, 0x65, 0x00, 0x6d, 0x00, 0x35, 0x00, 0x2f, 0x00, 0x3e}, StandardCharsets.UTF_16BE) + "')";
+        final String result = existEmbeddedServer.executeOneValue(query);
+        assertEquals("<elem5/>", result);
+    }
+
+    @Test
+    public void parseFragmentWithUtf16LEBomWithoutDecl() throws XMLDBException {
+        final String query = "parse-xml-fragment('" + UTF16_LE_BOM + new String(new byte[]{0x3c, 0x00, 0x65, 0x00, 0x6c, 0x00, 0x65, 0x00, 0x6d, 0x00, 0x36, 0x00, 0x2f, 0x00, 0x3e, 0x00}, StandardCharsets.UTF_16LE) + "')";
+        final String result = existEmbeddedServer.executeOneValue(query);
+        assertEquals("<elem6/>", result);
+    }
+}