HTML 5 namespace prefix normalization for HTML and XHTML serializers (#2588)

* fix 'canonical' handling for DBNode element namespace

* add HTML 5 namespace normalization to HTML and XHTML serializers

Author: GuntherRademacher

basex-core/src/main/java/org/basex/io/serial/HTMLSerializer.java

@@ -18,7 +18,7 @@
  * @author BaseX Team, BSD License
  * @author Christian Gruen
  */
-final class HTMLSerializer extends MarkupSerializer {
+final class HTMLSerializer extends XhtmlHtmlSerializer {
   /** (X)HTML: elements with an empty content model. */
   static final TokenSet EMPTIES = new TokenSet("area", "base", "basefont", "br", "col", "embed",
       "frame", "hr", "img", "input", "isindex", "link", "meta", "param");

basex-core/src/main/java/org/basex/io/serial/Serializer.java

@@ -62,9 +62,11 @@ public abstract class Serializer implements Closeable {
    * @param uri URI (can be {@code null})
    * @param value attribute value (can be {@code null})
    */
-  private record Att(byte[] name, byte[] value, byte[] uri) { }
-  /** Attribute/namespace collector. */
-  private final ArrayList<Att> attributes = new ArrayList<>();
+  protected record Att(byte[] name, byte[] value, byte[] uri) { }
+  /** Attribute collector. */
+  protected final ArrayList<Att> attributes = new ArrayList<>();
+  /** Namespace collector. */
+  protected final ArrayList<Att> namespaces = new ArrayList<>();
 
   /** Static context. */
   protected StaticContext sc;
@@ -270,6 +272,15 @@ protected boolean skipElement(final XNode node) {
   protected void attribute(final byte[] name, final byte[] value, final boolean standalone)
       throws IOException { }
 
+  /**
+   * Returns the element name (may be overridden to modify names).
+   * @param name original name
+   * @return modified name
+   */
+  protected QNm elementName(final QNm name) {
+    return name;
+  }
+
   /**
    * Starts an element.
    * @param name element name
@@ -278,6 +289,13 @@ protected void attribute(final byte[] name, final byte[] value, final boolean st
   @SuppressWarnings("unused")
   protected void startOpen(final QNm name) throws IOException { }
 
+  /**
+   * Adjusts namespaces before serializing namespaces and attributes.
+   * @param name original element name
+   */
+  @SuppressWarnings("unused")
+  protected void adjustNamespaces(final QNm name) { }
+
   /**
    * Finishes an opening element node.
    * @throws IOException I/O exception
@@ -358,8 +376,8 @@ private void addAttribute(final byte[] name, final byte[] value, final byte[] ur
    * @param prefix prefix
    * @param uri URI
    */
-  private void addNamespace(final byte[] prefix, final byte[] uri) {
-    attributes.add(new Att(prefix, null, uri));
+  protected void addNamespace(final byte[] prefix, final byte[] uri) {
+    namespaces.add(new Att(prefix, null, uri));
   }
 
   /**
@@ -384,10 +402,10 @@ private void emitAttributes() throws IOException {
    */
   private void emitNamespaces() throws IOException {
     if(canonical) {
-      attributes.sort((a, b) -> compare(a.name, b.name));
+      namespaces.sort((a, b) -> compare(a.name, b.name));
     }
-    for(final Att att : attributes) namespace(att.name, att.uri, false);
-    attributes.clear();
+    for(final Att att : namespaces) namespace(att.name, att.uri, false);
+    namespaces.clear();
   }
 
   /**
@@ -454,9 +472,10 @@ private void node(final DBNode node) throws IOException {
           nsUri = data.nspaces.uri(data.uriId(pre, kind));
         }
         // open element, serialize namespace declaration if it's new
-        openElement(new QNm(name, nsUri));
+        final QNm originalName = new QNm(name, nsUri);
+        openElement(elementName(originalName));
         if(nsUri == null) nsUri = EMPTY;
-        namespace(nsPrefix, nsUri, false);
+        addNamespace(nsPrefix, nsUri);
 
         // database contains namespaces: add declarations
         if(nsExist) {
@@ -476,7 +495,6 @@ private void node(final DBNode node) throws IOException {
           } while(p >= 0 && data.kind(p) == Data.ELEM);
 
           // reset namespace cache
-          emitNamespaces();
           nsSet.clear();
         }
 
@@ -488,6 +506,8 @@ private void node(final DBNode node) throws IOException {
           addAttribute(n, v, canonical ? data.nspaces.uri(data.uriId(pre, Data.ATTR)) : null);
           if(eq(n, XML_SPACE) && indent) indent = !eq(v, PRESERVE);
         }
+        adjustNamespaces(originalName);
+        emitNamespaces();
         emitAttributes();
         parentStack.push(par);
       }
@@ -531,7 +551,8 @@ private void node(final FNode node) throws IOException {
       closeDoc();
     } else if(skip == 0 || !skipElement(node)) {
       // serialize elements (code will never be called for attributes)
-      final QNm name = node.qname();
+      final QNm originalName = node.qname();
+      final QNm name = elementName(originalName);
       openElement(name);
 
       // serialize declared namespaces
@@ -540,7 +561,6 @@ private void node(final FNode node) throws IOException {
       for(int p = 0; p < ps; p++) addNamespace(nsp.name(p), nsp.value(p));
       // add new or updated namespace
       addNamespace(name.prefix(), name.uri());
-      emitNamespaces();
 
       // serialize attributes
       final boolean i = indent;
@@ -550,6 +570,8 @@ private void node(final FNode node) throws IOException {
         addAttribute(n, v, canonical ? nsUri(prefix(n)) : null);
         if(eq(n, XML_SPACE) && indent) indent = !eq(v, PRESERVE);
       }
+      adjustNamespaces(originalName);
+      emitNamespaces();
       emitAttributes();
 
       // serialize children

basex-core/src/main/java/org/basex/io/serial/XHTMLSerializer.java

@@ -15,7 +15,7 @@
  * @author BaseX Team, BSD License
  * @author Christian Gruen
  */
-final class XHTMLSerializer extends MarkupSerializer {
+final class XHTMLSerializer extends XhtmlHtmlSerializer {
   /**
    * Constructor, specifying serialization options.
    * @param os output stream

basex-core/src/main/java/org/basex/io/serial/XhtmlHtmlSerializer.java

@@ -0,0 +1,61 @@
+package org.basex.io.serial;
+
+import static org.basex.util.Token.*;
+import static org.basex.util.XMLToken.*;
+
+import java.io.*;
+
+import org.basex.query.value.item.*;
+
+/**
+ * This class contains the common behavior of XHTML and HTML serializers.
+ *
+ * @author BaseX Team, BSD License
+ * @author Gunther Rademacher
+ */
+abstract class XhtmlHtmlSerializer extends MarkupSerializer {
+  /**
+   * Constructor.
+   * @param os output stream
+   * @param sopts serialization parameters
+   * @param versions supported versions
+   * @throws IOException I/O exception
+   */
+  protected XhtmlHtmlSerializer(final OutputStream os, final SerializerOptions sopts,
+      final String... versions) throws IOException {
+    super(os, sopts, versions);
+  }
+
+  @Override
+  protected final QNm elementName(final QNm name) {
+    return mustRewrite(name) ? new QNm(name.local(), name.uri()) : name;
+  }
+
+  @Override
+  protected final void adjustNamespaces(final QNm name) {
+    if(!mustRewrite(name)) return;
+    final byte[] prefix = name.prefix();
+    boolean unused = true;
+    for(final Att att : attributes) {
+      final byte[] an = att.name();
+      if(startsWith(an, prefix) && indexOf(an, ':') == prefix.length) {
+        unused = false;
+        break;
+      }
+    }
+    for(int i = namespaces.size() - 1; i >= 0; --i) {
+      final byte[] nsName = namespaces.get(i).name();
+      if(nsName.length == 0 || unused && eq(nsName, prefix)) namespaces.remove(i);
+    }
+    addNamespace(EMPTY, name.uri());
+  }
+
+  /**
+   * Checks if the namespace prefix of an element name must be rewritten.
+   * @param name name to be checked
+   * @return result of check
+   */
+  private boolean mustRewrite(final QNm name) {
+    return html5 && name.hasPrefix() && eq(name.uri(), XHTML_URI, MATHML_URI, SVG_URI);
+  }
+}

basex-core/src/main/java/org/basex/util/XMLToken.java

@@ -113,6 +113,10 @@ public final class XMLToken {
   public static final byte[] XHTML_URI = token("http://www.w3.org/1999/xhtml");
   /** XML namespace. */
   public static final byte[] XML_URI = token("http://www.w3.org/XML/1998/namespace");
+  /** MathML namespace. */
+  public static final byte[] MATHML_URI = token("http://www.w3.org/1998/Math/MathML");
+  /** SVG namespace. */
+  public static final byte[] SVG_URI    = token("http://www.w3.org/2000/svg");
 
   /** Index for all HTML entities (lazy initialization). */
   private static TokenObjectMap<byte[]> entities;

basex-core/src/test/java/org/basex/query/SerializerTest.java

@@ -175,6 +175,19 @@ public final class SerializerTest extends SandboxTest {
     }
     query(option + "<a>&#x90;</a>", "<a>&#x90;</a>");
     query(option + "<html/>", "<!DOCTYPE HTML><html></html>");
+    query("declare namespace xhtml = 'http://www.w3.org/1999/xhtml';\n"
+        + option + INDENT_ATTRIBUTES.arg("yes") + INDENT.arg("yes")
+        + "<html>\n"
+        + "  <xhtml:body xhtml:test='x' xmlns='http://www.w3.org/2000/svg'>\n"
+        + "     <svg/>\n"
+        + "  </xhtml:body>\n"
+        + "</html>",
+        "<!DOCTYPE HTML>\n"
+        + "<html>\n"
+        + "  <body xmlns:xhtml=\"http://www.w3.org/1999/xhtml\"\n"
+        + "        xmlns=\"http://www.w3.org/1999/xhtml\"\n"
+        + "        xhtml:test=\"x\"><svg xmlns=\"http://www.w3.org/2000/svg\"/></body>\n"
+        + "</html>");
   }
 
   /** Test: method=text. */

basex-core/src/test/java/org/basex/query/func/FnModuleTest.java

@@ -3022,6 +3022,9 @@ public final class FnModuleTest extends SandboxTest {
     final String canonicalXml = " { 'method': 'xml', 'canonical': true() }";
     query(func.args(" <a xmlns:p='urn:test:x' p:z='1' a='2'/>", canonicalXml),
         "<a xmlns:p=\"urn:test:x\" a=\"2\" p:z=\"1\"></a>");
+    query(func.args(" parse-xml(`<q:a xmlns:p='urn:test:x' xmlns:q='urn:test:y' p:z='1' a='2'/>`)",
+        canonicalXml),
+        "<q:a xmlns:p=\"urn:test:x\" xmlns:q=\"urn:test:y\" a=\"2\" p:z=\"1\"></q:a>");
 
     final String canonicalJson = " {'method': 'json', 'canonical': true()}";
     query(func.args(" [0x7fff_ffff_ffff_ffff]", canonicalJson), "[9223372036854776000]");