Merge pull request #4547 from evolvedbinary/feature/doctype-serialization

Enable Document Type Declaration Serialization

Author: reinhapa

exist-core/src/main/java/org/exist/client/InteractiveClient.java

@@ -84,6 +84,7 @@
 import se.softhouse.jargo.ArgumentException;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.exist.storage.serializers.EXistOutputKeys.OUTPUT_DOCTYPE;
 
 /**
  * Command-line client based on the XML:DB API.
@@ -137,6 +138,7 @@ public class InteractiveClient {
         defaultProps.setProperty(USER, USER_DEFAULT);
         defaultProps.setProperty(EDITOR, EDIT_CMD);
         defaultProps.setProperty(INDENT, "true");
+        defaultProps.setProperty(OUTPUT_DOCTYPE, "true");
         defaultProps.setProperty(ENCODING, ENCODING_DEFAULT.name());
         defaultProps.setProperty(COLORS, "false");
         defaultProps.setProperty(PERMISSIONS, "false");

exist-core/src/main/java/org/exist/http/RESTServer.java

@@ -250,6 +250,9 @@ private String getParameter(final HttpServletRequest request, final RESTServerPa
      *
      * <li>_xsl: an URI pointing to an XSL stylesheet that will be applied to
      * the returned XML.</li>
+     *
+     * <li>_output-doctype: if set to "yes", the returned XML will include
+     * a Document Type Declaration if one is present, if "no" the Document Type Declaration will be omitted.</li>
      * </ul>
      *
      * @param broker the database broker
@@ -346,6 +349,14 @@ public void doGet(final DBBroker broker, final Txn transaction, final HttpServle
         if ((option = getParameter(request, Indent)) != null) {
             outputProperties.setProperty(OutputKeys.INDENT, option);
         }
+        if ((option = getParameter(request, Output_Doctype)) != null) {
+            // take user query-string specified output-doctype setting
+            outputProperties.setProperty(EXistOutputKeys.OUTPUT_DOCTYPE, option);
+        } else {
+            // set output-doctype by configuration
+            final String outputDocType = broker.getConfiguration().getProperty(Serializer.PROPERTY_OUTPUT_DOCTYPE, "yes");
+            outputProperties.setProperty(EXistOutputKeys.OUTPUT_DOCTYPE, outputDocType);
+        }
         if ((option = getParameter(request, Source)) != null && !safeMode) {
             source = "yes".equals(option);
         }

exist-core/src/main/java/org/exist/http/RESTServerParameter.java

@@ -346,7 +346,18 @@ enum RESTServerParameter {
      * <exist:property name = string
      *  value = string/>
      */
-    Property;
+    Property,
+
+    /**
+     * Can be used in the Query String of a GET request
+     * to indicate that the doctype of an XML document should also
+     * be serialized if present.
+     *
+     * Contexts: GET
+     *
+     * The value of the parameter should be either "yes" or "no".
+     */
+    Output_Doctype;
 
     /**
      * Get the parameter key that is
@@ -366,6 +377,6 @@ public String queryStringKey() {
      * @return The parameter key, suitable for use in an XML document
      */
     public String xmlKey() {
-        return name().toLowerCase();
+        return name().toLowerCase().replace('_', '-');
     }
 }

exist-core/src/main/java/org/exist/protocolhandler/xmlrpc/XmlrpcDownload.java

@@ -36,6 +36,9 @@
 import org.apache.xmlrpc.client.XmlRpcClientConfigImpl;
 
 import org.exist.protocolhandler.xmldb.XmldbURL;
+import org.exist.storage.serializers.EXistOutputKeys;
+
+import javax.xml.transform.OutputKeys;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 
@@ -75,8 +78,9 @@ public void stream(final XmldbURL xmldbURL, final OutputStream os) throws IOExce
 
             // Setup xml serializer
             final Map<String, String> options = new HashMap<>();
-            options.put("indent", "no");
-            options.put("encoding", UTF_8.name());
+            options.put(OutputKeys.INDENT, "no");
+            options.put(OutputKeys.ENCODING, UTF_8.name());
+            options.put(EXistOutputKeys.OUTPUT_DOCTYPE, "yes");  // Preserve doctype
 
             // Setup client parameters
             final List<Object> params = new ArrayList<>();

exist-core/src/main/java/org/exist/storage/serializers/Serializer.java

@@ -118,6 +118,8 @@ public abstract class Serializer implements XMLReader {
 	protected final static Logger LOG = LogManager.getLogger(Serializer.class);
 
 	public static final String CONFIGURATION_ELEMENT_NAME = "serializer";
+	public static final String OUTPUT_DOCTYPE_ATTRIBUTE = "output-doctype";
+	public static final String PROPERTY_OUTPUT_DOCTYPE = "serialization.output-doctype";
 	public static final String ENABLE_XINCLUDE_ATTRIBUTE = "enable-xinclude";
 	public static final String PROPERTY_ENABLE_XINCLUDE = "serialization.enable-xinclude";
 	public static final String ENABLE_XSL_ATTRIBUTE = "enable-xsl";

exist-core/src/main/java/org/exist/util/Configuration.java

@@ -652,6 +652,12 @@ private void configureHtmlToXmlParser(final Element parser) {
      */
     private void configureSerializer( Element serializer )
     {
+        final String outputDocType = getConfigAttributeValue( serializer, Serializer.OUTPUT_DOCTYPE_ATTRIBUTE );
+        if (outputDocType != null) {
+            config.put(Serializer.PROPERTY_OUTPUT_DOCTYPE, outputDocType);
+            LOG.debug(Serializer.PROPERTY_OUTPUT_DOCTYPE + ": {}", config.get(Serializer.PROPERTY_OUTPUT_DOCTYPE));
+        }
+
         final String xinclude = getConfigAttributeValue( serializer, Serializer.ENABLE_XINCLUDE_ATTRIBUTE );
 
         if( xinclude != null ) {

exist-core/src/main/java/org/exist/xmldb/LocalCollection.java

@@ -89,6 +89,7 @@ public class LocalCollection extends AbstractLocal implements EXistCollection {
         defaultProperties.setProperty(EXistOutputKeys.EXPAND_XINCLUDES, "yes");
         defaultProperties.setProperty(EXistOutputKeys.PROCESS_XSL_PI, "no");
         defaultProperties.setProperty(NORMALIZE_HTML, "no");
+        defaultProperties.setProperty(EXistOutputKeys.OUTPUT_DOCTYPE, "yes");
     }
 
     private final XmldbURI path;

exist-core/src/main/java/org/exist/xmlrpc/RpcConnection.java

@@ -663,6 +663,11 @@ private String getDocumentAsString(final XmldbURI docUri, final Map<String, Obje
     }
 
     private void serialize(final DBBroker broker, final Properties properties, final ConsumerE<Serializer, SAXException> toSaxFunction, final Writer writer) throws SAXException, IOException {
+        if (!properties.containsKey(EXistOutputKeys.OUTPUT_DOCTYPE)) {
+            final String outputDocType = broker.getConfiguration().getProperty(Serializer.PROPERTY_OUTPUT_DOCTYPE, "yes");
+            properties.setProperty(EXistOutputKeys.OUTPUT_DOCTYPE, outputDocType);
+        }
+
         final Serializer serializer = broker.borrowSerializer();
 
         SAXSerializer saxSerializer = null;

exist-core/src/main/java/org/exist/xquery/functions/fn/FunSerialize.java

@@ -131,12 +131,14 @@ private static boolean isSerializationParametersElement(final Item item) {
      * <a href="http://www.w3.org/TR/xslt-xquery-serialization-30/#serdm">XSLT and XQuery Serialization 3.0 - Sequence Normalization</a>.
      *
      * @param callingExpr the expression from which the function is called.
-     *                    needed for error reporting
-     * @param context current context
-     * @param input non-normalized sequence
-     * @param itemSeparator placed between each item
-     * @return normalized sequence
-     * @throws XPathException in case of dynamic error
+     *                    Needed for error reporting
+     * @param context current context.
+     * @param input non-normalized sequence.
+     * @param itemSeparator the item separator placed between each item.
+     *
+     * @return normalized sequence.
+     *
+     * @throws XPathException in case of dynamic error.
      */
     public static Sequence normalize(final Expression callingExpr, final XQueryContext context, final Sequence input, final String itemSeparator) throws XPathException {
         if (input.isEmpty())

exist-core/src/main/java/org/exist/xquery/util/SerializerUtils.java

@@ -185,16 +185,18 @@ public Sequence getDefaultValue() {
      * for Exist xquery specific functions
      */
     public enum ExistParameterConvention implements ParameterConvention<QName> {
-        EXPAND_XINCLUDE("expand-xincludes", Type.BOOLEAN, Cardinality.ZERO_OR_ONE, BooleanValue.TRUE),
-        PROCESS_XSL_PI("process-xsl-pi", Type.BOOLEAN, Cardinality.ZERO_OR_ONE, BooleanValue.TRUE),
-        JSON_IGNORE_WHITE_SPACE_TEXT_NODES("json-ignore-whitespace-text-nodes", Type.BOOLEAN, Cardinality.ZERO_OR_ONE, BooleanValue.TRUE),
-        HIGHLIGHT_MATCHES("highlight-matches", Type.STRING, Cardinality.ZERO_OR_ONE, new StringValue("none")),
-        JSONP("jsonp", Type.STRING, Cardinality.ZERO_OR_ONE, Sequence.EMPTY_SEQUENCE),
-        ADD_EXIST_ID("add-exist-id", Type.STRING, Cardinality.ZERO_OR_ONE, new StringValue("none")),
+        OUTPUT_DOCTYPE(EXistOutputKeys.OUTPUT_DOCTYPE, Type.BOOLEAN, Cardinality.ZERO_OR_ONE, BooleanValue.FALSE),
+        EXPAND_XINCLUDE(EXistOutputKeys.EXPAND_XINCLUDES, Type.BOOLEAN, Cardinality.ZERO_OR_ONE, BooleanValue.TRUE),
+        PROCESS_XSL_PI(EXistOutputKeys.PROCESS_XSL_PI, Type.BOOLEAN, Cardinality.ZERO_OR_ONE, BooleanValue.TRUE),
+        JSON_IGNORE_WHITE_SPACE_TEXT_NODES(EXistOutputKeys.JSON_IGNORE_WHITESPACE_TEXT_NODES, Type.BOOLEAN, Cardinality.ZERO_OR_ONE, BooleanValue.TRUE),
+        HIGHLIGHT_MATCHES(EXistOutputKeys.HIGHLIGHT_MATCHES, Type.STRING, Cardinality.ZERO_OR_ONE, new StringValue("none")),
+        JSONP(EXistOutputKeys.JSONP, Type.STRING, Cardinality.ZERO_OR_ONE, Sequence.EMPTY_SEQUENCE),
+        ADD_EXIST_ID(EXistOutputKeys.ADD_EXIST_ID, Type.STRING, Cardinality.ZERO_OR_ONE, new StringValue("none")),
         // default of 4 corresponds to the existing eXist default, although 3 is in the spec
         INDENT_SPACES("indent-spaces", Type.INTEGER, Cardinality.ZERO_OR_ONE, new IntegerValue(4)),
         INSERT_FINAL_NEWLINE(EXistOutputKeys.INSERT_FINAL_NEWLINE, Type.BOOLEAN, Cardinality.ZERO_OR_ONE, BooleanValue.FALSE);
 
+
         private final QName parameterName;
         private final int type;
         private final Cardinality cardinality;