Bring self-closing tags into HTML5 spec

For #2300

In HTML, only foreign elements (svg, math) can self close.

Allows users to override via custom tags.

XML parse and syntax will allow self-closing.

Author: jhy

CHANGES.md

@@ -4,8 +4,17 @@
 
 ### Changes
 
+* To better follow the HTML5 spec and current browsers, the HTML parser no longer allows self-closing tags (`<foo />`)
+  to close HTML elements by default. Foreign content (SVG, MathML), and content parsed with the XML parser, still
+  supports self-closing tags. If you need specific HTML tags to support self-closing, you can register a custom tag via
+  the `TagSet` configured in `Parser.tagSet()`, using `Tag#set(Tag.SelfClose)`. Void/empty tags (like `<img>`) are
+  unaffected by this change. [#2300](https://github.com/jhy/jsoup/issues/2300).
+
 ### Improvements
 
+* Added the ability to define custom tags, and to modify properties of known tags, via the `TagSet` tag collection.
+  Their properties can impact both the parse and how content is
+  serialized. [#2285](https://github.com/jhy/jsoup/issues/2285).
 * `Element.cssSelector()` will prefer to return shorter selectors by using ancestor IDs when available and unique. E.g.
   `#id > div > p` instead of  `html > body > div > div > p` [#2283](https://github.com/jhy/jsoup/pull/2283).
 * Added `Elements.deselect(int index)`, `Elements.deselect(Object o)`, and `Elements.deselectAll()` methods to remove

src/main/java/org/jsoup/nodes/Element.java

@@ -1812,25 +1812,29 @@ public Range endSourceRange() {
 
     @Override
     void outerHtmlHead(final Appendable accum, Document.OutputSettings out) throws IOException {
-        accum.append('<').append(safeTagName(out.syntax()));
+        String tagName = safeTagName(out.syntax());
+        accum.append('<').append(tagName);
         if (attributes != null) attributes.html(accum, out);
 
-        // selfclosing includes unknown tags, isEmpty defines tags that are always empty
-        if (childNodes.isEmpty() && tag.isSelfClosing()) {
-            if (out.syntax() == html && tag.isEmpty())
+        if (childNodes.isEmpty()) {
+            boolean xmlMode = out.syntax() == xml || !tag.namespace().equals(NamespaceHtml);
+            if (xmlMode && (tag.is(Tag.SeenSelfClose) || (tag.isKnownTag() && (tag.isEmpty() || tag.isSelfClosing())))) {
+                accum.append(" />");
+            } else if (!xmlMode && tag.isEmpty()) { // html void element
                 accum.append('>');
-            else
-                accum.append(" />"); // <img> in html, <img /> in xml
-        }
-        else
+            } else {
+                accum.append("></").append(tagName).append('>');
+            }
+        } else {
             accum.append('>');
+        }
     }
 
     @Override
     void outerHtmlTail(Appendable accum, Document.OutputSettings out) throws IOException {
-        if (!(childNodes.isEmpty() && tag.isSelfClosing())) {
+        if (!childNodes.isEmpty())
             accum.append("</").append(safeTagName(out.syntax())).append('>');
-        }
+        // if empty, we have already closed in htmlHead
     }
 
     /* If XML syntax, normalizes < to _ in tag name. */

src/main/java/org/jsoup/parser/HtmlTreeBuilder.java

@@ -328,21 +328,18 @@ Element insertElementFor(final Token.StartTag startTag) {
         Element el = createElementFor(startTag, NamespaceHtml, false);
         doInsertElement(el);
 
-        // handle self-closing tags. when the spec expects an empty tag, will directly hit insertEmpty, so won't generate this fake end tag.
+        // handle self-closing tags. when the spec expects an empty (void) tag, will directly hit insertEmpty, so won't generate this fake end tag.
         if (startTag.isSelfClosing()) {
             Tag tag = el.tag();
-            if (tag.isKnownTag()) {
-                if (!tag.isEmpty())
-                    tokeniser.error("Tag [%s] cannot be self closing; not a void tag", tag.normalName());
-                // else: ok
+            tag.setSeenSelfClose(); // can infer output if in xml syntax
+            if (tag.isKnownTag() && (tag.isEmpty() || tag.isSelfClosing())) {
+                // ok, allow it. effectively a pop, but fiddles with the state. handles empty style, title etc which would otherwise leave us in data state
+                tokeniser.transition(TokeniserState.Data); // handles <script />, otherwise needs breakout steps from script data
+                tokeniser.emit(emptyEnd.reset().name(el.tagName()));  // ensure we get out of whatever state we are in. emitted for yielded processing
+            } else {
+                // error it, and leave the inserted element on
+                tokeniser.error("Tag [%s] cannot be self-closing; not a void tag", tag.normalName());
             }
-            else { // unknown tag: remember this is self-closing, for output
-                tag.setSelfClosing();
-            }
-
-            // effectively a pop, but fiddles with the state. handles empty style, title etc which would otherwise leave us in data state
-            tokeniser.transition(TokeniserState.Data); // handles <script />, otherwise needs breakout steps from script data
-            tokeniser.emit(emptyEnd.reset().name(el.tagName()));  // ensure we get out of whatever state we are in. emitted for yielded processing
         }
 
         return el;
@@ -355,8 +352,8 @@ Element insertForeignElementFor(final Token.StartTag startTag, String namespace)
         Element el = createElementFor(startTag, namespace, true);
         doInsertElement(el);
 
-        if (startTag.isSelfClosing()) {
-            el.tag().setSelfClosing(); // remember this is self-closing for output
+        if (startTag.isSelfClosing()) { // foreign els are OK to self-close
+            el.tag().setSeenSelfClose(); // remember this is self-closing for output
             pop();
         }
 

src/main/java/org/jsoup/parser/Tag.java

@@ -324,9 +324,8 @@ public boolean isFormSubmittable() {
         return (options &= FormSubmittable) != 0;
     }
 
-    Tag setSelfClosing() {
-        options |= Tag.SelfClose; // does not change known status. // todo will refactor how self-closing is handled in TreeBuilder
-        return this;
+    void setSeenSelfClose() {
+        options |= Tag.SeenSelfClose; // does not change known status
     }
 
     /**

src/main/java/org/jsoup/parser/TreeBuilder.java

@@ -52,6 +52,7 @@ void initialiseParse(Reader input, String baseUri, Parser parser) {
         reader = new CharacterReader(input);
         trackSourceRange = parser.isTrackPosition();
         reader.trackNewlines(parser.isTrackErrors() || trackSourceRange); // when tracking errors or source ranges, enable newline tracking for better legibility
+        if (parser.isTrackErrors()) parser.getErrors().clear();
         tokeniser = new Tokeniser(this);
         stack = new ArrayList<>(32);
         tagSet = parser.tagSet();

src/main/java/org/jsoup/parser/XmlTreeBuilder.java

@@ -157,7 +157,7 @@ void insertElementFor(Token.StartTag startTag) {
         push(el);
 
         if (startTag.isSelfClosing()) {
-            tag.setSelfClosing();
+            tag.setSeenSelfClose();
             pop(); // push & pop ensures onNodeInserted & onNodeClosed
         } else {
             TokeniserState textState = tag.textState();

src/test/java/org/jsoup/nodes/DocumentTest.java

@@ -7,6 +7,7 @@
 import org.jsoup.nodes.Document.OutputSettings.Syntax;
 import org.jsoup.parser.ParseSettings;
 import org.jsoup.parser.Parser;
+import org.jsoup.parser.Tag;
 import org.jsoup.select.Elements;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
@@ -158,14 +159,16 @@ public class DocumentTest {
 
     @Test public void testHtmlAndXmlSyntax() {
         String h = "<!DOCTYPE html><body><img async checked='checked' src='&<>\"'>&lt;&gt;&amp;&quot;<foo />bar";
-        Document doc = Jsoup.parse(h);
+        Parser parser = Parser.htmlParser();
+        parser.tagSet().valueOf("foo", Parser.NamespaceHtml).set(Tag.SelfClose); // customize foo to allow self close
+        Document doc = Jsoup.parse(h, parser);
 
         doc.outputSettings().syntax(Syntax.html);
         assertEquals("<!doctype html>\n" +
                 "<html>\n" +
                 " <head></head>\n" +
                 " <body>\n" +
-                "  <img async checked src=\"&amp;<>&quot;\">&lt;&gt;&amp;\"<foo />bar\n" +
+                "  <img async checked src=\"&amp;<>&quot;\">&lt;&gt;&amp;\"<foo></foo>bar\n" + // html won't include self-closing
                 " </body>\n" +
                 "</html>", doc.html());
 
@@ -174,7 +177,7 @@ public class DocumentTest {
                 "<html>\n" +
                 " <head></head>\n" +
                 " <body>\n" +
-                "  <img async=\"\" checked=\"checked\" src=\"&amp;&lt;>&quot;\" />&lt;&gt;&amp;\"<foo />bar\n" +
+                "  <img async=\"\" checked=\"checked\" src=\"&amp;&lt;>&quot;\" />&lt;&gt;&amp;\"<foo />bar\n" + // xml will
                 " </body>\n" +
                 "</html>", doc.html());
     }

src/test/java/org/jsoup/nodes/ElementTest.java

@@ -2555,9 +2555,9 @@ void prettySerializationRoundTrips(Document.OutputSettings settings) {
         Document doc = Jsoup.parse(html);
         assertEquals("<bar>\n <p></p>\n</bar>", doc.body().html());
 
-        html = "<foo>\n  <bar />\n</foo>";
+        html = "<foo>\n  <bar /></foo>";
         doc = Jsoup.parse(html);
-        assertEquals("<foo>\n <bar />\n</foo>", doc.body().html());
+        assertEquals("<foo>\n <bar></bar>\n</foo>", doc.body().html());
     }
 
     @Test void spanInBlockTrims() {