Merge pull request #4264 from eXistSolutions/feat/insert-final-newline

dizzzz · web-flow · commit db713cae64d8 · 2022-08-08T20:11:09.000+02:00
[enhancement] add serialization option that inserts final newline
diff --git a/exist-core/src/main/java/org/exist/storage/serializers/EXistOutputKeys.java b/exist-core/src/main/java/org/exist/storage/serializers/EXistOutputKeys.java
@@ -104,4 +104,14 @@ public class EXistOutputKeys {
      * Set to "yes" to enable xdm-serialization rules, false otherwise.
      */
     public final static String XDM_SERIALIZATION = "xdm-serialization";
+
+    /**
+     * Enforce newline at the end of JSON and XML documents.
+     *
+     * It is common for editor software to enforce a newline at the end of non-
+     * binary resources. This setting will do the same for serialization out of
+     * exist-db and will lead to less meaningless changes in git and  tools
+     * like diff will be able to provide more meaningful information as well.
+     */
+    public final static String INSERT_FINAL_NEWLINE = "insert-final-newline";
 }
diff --git a/exist-core/src/main/java/org/exist/util/serializer/IndentingXMLWriter.java b/exist-core/src/main/java/org/exist/util/serializer/IndentingXMLWriter.java
@@ -140,6 +140,14 @@ public void processingInstruction(final String target, final String data) throws
         afterTag = true;
     }
 
+    @Override
+    public void endDocument() throws TransformerException {
+        super.endDocument();
+        if ("yes".equals(outputProperties.getProperty(EXistOutputKeys.INSERT_FINAL_NEWLINE, "no"))) {
+            super.characters("\n");
+        }
+    }
+
     @Override
     public void endDocumentType() throws TransformerException {
         super.endDocumentType();
diff --git a/exist-core/src/main/java/org/exist/util/serializer/json/JSONSerializer.java b/exist-core/src/main/java/org/exist/util/serializer/json/JSONSerializer.java
@@ -70,6 +70,9 @@ public void serialize(Sequence sequence, Writer writer) throws SAXException {
                 generator.disable(JsonGenerator.Feature.STRICT_DUPLICATE_DETECTION);
             }
             serializeSequence(sequence, generator);
+            if ("yes".equals(outputProperties.getProperty(EXistOutputKeys.INSERT_FINAL_NEWLINE, "no"))) {
+                generator.writeRaw('\n');
+            }
             generator.close();
         } catch (IOException | XPathException e) {
             throw new SAXException(e.getMessage(), e);
diff --git a/exist-core/src/main/java/org/exist/xquery/util/SerializerUtils.java b/exist-core/src/main/java/org/exist/xquery/util/SerializerUtils.java
@@ -163,7 +163,8 @@ public enum ExistParameterConvention implements ParameterConvention<QName> {
         JSON_IGNORE_WHITE_SPACE_TEXT_NODES("json-ignore-whitespace-text-nodes", Type.BOOLEAN, Cardinality.ZERO_OR_ONE, BooleanValue.TRUE),
         HIGHLIGHT_MATCHES("highlight-matches", Type.STRING, Cardinality.ZERO_OR_ONE, new StringValue("none")),
         JSONP("jsonp", Type.STRING, Cardinality.ZERO_OR_ONE, Sequence.EMPTY_SEQUENCE),
-        ADD_EXIST_ID("add-exist-id", Type.STRING, Cardinality.ZERO_OR_ONE, new StringValue("none"));
+        ADD_EXIST_ID("add-exist-id", Type.STRING, Cardinality.ZERO_OR_ONE, new StringValue("none")),
+        ADD_NEWLINE_AT_EOF(EXistOutputKeys.INSERT_FINAL_NEWLINE, Type.BOOLEAN, Cardinality.ZERO_OR_ONE, BooleanValue.FALSE);
 
 
         private final QName parameterName;
diff --git a/exist-core/src/test/xquery/xquery3/serialize.xql b/exist-core/src/test/xquery/xquery3/serialize.xql
@@ -846,3 +846,51 @@ function ser:serialize-xml-134() {
     }
     return serialize((1 to 4)!text{.}, $params)
 };
+
+declare
+    %test:assertTrue
+function ser:exist-insert-final-newline-false() {
+    let $doc := <root>
+    <nested />
+    </root>
+    let $serialized := fn:serialize($doc,
+        map {xs:QName("exist:insert-final-newline"): false()})
+    return fn:ends-with($serialized, ">")
+};
+
+declare
+    %test:assertTrue
+function ser:exist-insert-final-newline-true() {
+    let $doc := <root>
+    <nested />
+    </root>
+    let $serialized := fn:serialize($doc,
+        map {xs:QName("exist:insert-final-newline"): true()})
+    return fn:ends-with($serialized, "&#x0A;")
+};
+
+declare
+    %test:assertTrue
+function ser:exist-insert-final-newline-false-json() {
+    let $doc := map { "a": 1 }
+    let $serialized := fn:serialize($doc,
+        map {
+            "method": "json",
+            "exist:insert-final-newline": false()
+        }
+    )
+    return fn:ends-with($serialized, "}")
+};
+
+declare
+    %test:assertTrue
+function ser:exist-insert-final-newline-true-json() {
+    let $doc := map { "a": 1 }
+    let $serialized := fn:serialize($doc,
+        map {
+            "method": "json",
+            "exist:insert-final-newline": true()
+        }
+    )
+    return fn:ends-with($serialized, "&#x0A;")
+};
diff --git a/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/Sync.java b/extensions/modules/file/src/main/java/org/exist/xquery/modules/file/Sync.java
@@ -178,8 +178,8 @@ private Map<String, Sequence> getOptions(final Sequence parameter) throws XPathE
             }
             options.put(EXCLUDES_OPT, seq);
 
-            checkOption(optionsMap, PRUNE_OPT, options, Type.BOOLEAN);
-            checkOption(optionsMap, AFTER_OPT, options, Type.DATE_TIME, Type.DATE_TIME_STAMP);
+            checkOption(optionsMap, PRUNE_OPT, Type.BOOLEAN, options);
+            checkOption(optionsMap, AFTER_OPT, Type.DATE_TIME, options);
         } else if (parameter.itemAt(0).getType() == Type.DATE_TIME) {
             options.put(AFTER_OPT, parameter);
         } else {
@@ -192,43 +192,25 @@ private Map<String, Sequence> getOptions(final Sequence parameter) throws XPathE
     private void checkOption(
             final AbstractMapType optionsMap,
             final String name,
-            final Map<String, Sequence> options,
-            final int... expectedTypes
+            final int expectedType,
+            final Map<String, Sequence> options
     ) throws XPathException {
         final Sequence p = optionsMap.get(new StringValue(this, name));
 
         if (p.isEmpty()) {
             return; // nothing to do, continue
         }
 
-        if (p.hasMany() || !isExpectedType(p.getItemType(), expectedTypes)) {
+        if (p.hasMany() || !Type.subTypeOf(p.getItemType(),expectedType)) {
             throw new XPathException(this, ErrorCodes.XPTY0004,
                     "Invalid value type for option \"" + name + "\", expected " +
-                            formatTypes(expectedTypes) + " got " +
+                            Type.getTypeName(expectedType) + " got " +
                             Type.getTypeName(p.itemAt(0).getType()));
         }
 
         options.put(name, p);
     }
 
-    private boolean isExpectedType(final int actualType, final int... expectedTypes) {
-        for(int type : expectedTypes) {
-            if(type == actualType) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    private String formatTypes(final int... expectedTypes) {
-        StringBuilder buff = new StringBuilder(Type.getTypeName(expectedTypes[0]));
-        for(int i = 1; i < expectedTypes.length; i++) {
-            buff.append(", ");
-            buff.append(Type.getTypeName(expectedTypes[i]));
-        }
-        return buff.toString();
-    }
-
     private Sequence startSync(
             final String target,
             final String collectionPath,
diff --git a/extensions/modules/file/src/test/xquery/modules/file/serialize.xqm b/extensions/modules/file/src/test/xquery/modules/file/serialize.xqm
@@ -23,40 +23,126 @@ xquery version "3.1";
 
 module namespace serialize="http://exist-db.org/testsuite/modules/file/serialize";
 
-import module namespace test="http://exist-db.org/xquery/xqsuite" at "resource:org/exist/xquery/lib/xqsuite/xqsuite.xql";
 import module namespace file="http://exist-db.org/xquery/file";
+import module namespace helper="http://exist-db.org/xquery/test/util/helper" at "resource:util/helper.xqm";
+import module namespace fixtures="http://exist-db.org/xquery/test/util/fixtures" at "resource:util/fixtures.xqm";
+
+
+declare namespace test="http://exist-db.org/xquery/xqsuite";
+
+
+declare variable $serialize:suite := "serialize";
+declare variable $serialize:text := <node>data</node>;
+declare variable $serialize:xml := <root>
+    <unary attribute="value" />
+    <nested>
+        text
+    </nested>
+</root>;
 
 
 declare
-    %test:pending("need to mechanism to setup a temporary file to work with")
-    %test:assertEquals("datadata", "true", "true", "true")
+    %test:tearDown
+function serialize:tear-down() as empty-sequence() {
+    helper:clear-suite-fs($serialize:suite)
+};
+
+declare
+    %test:assertEquals("datamoredata")
 function serialize:append() {
+    let $directory := helper:get-test-directory($serialize:suite)
+    let $_ := file:mkdirs($directory)
+
+    let $path := $directory || "/append-test.txt"
 
-    let $node-set := text {"data"}
-    let $path := system:get-exist-home() || "/test.txt"
-    let $parameters := ()
-    let $append := true()
-    let $remove := file:delete($path)
-    let $ser1 := 	file:serialize($node-set, $path, (), false())
-    let $ser2 := 	file:serialize($node-set, $path, (), true())
-    let $read := file:read($path)
-    let $remove := file:delete($path)
-    return ($read, $ser1, $ser2, $remove)
+    let $_ := file:serialize-binary(xs:base64Binary("data"), $path)
+    let $_ := file:serialize-binary(xs:base64Binary("moredata"), $path, true())
+
+    return file:read-binary($path) => xs:string()
 };
 
 declare
-    %test:pending("need to mechanism to setup a temporary file to work with")
-    %test:assertEquals("data", "true", "true", "true")
+    %test:assertEquals("moredata")
 function serialize:overwrite() {
+    let $directory := helper:get-test-directory($serialize:suite)
+    let $_ := file:mkdirs($directory)
+
+    let $path := $directory || "/overwrite-test.txt"
+    let $_ := file:serialize-binary(xs:base64Binary("data"), $path)
+    let $_ := file:serialize-binary(xs:base64Binary("moredata"), $path, false())
+
+    return file:read-binary($path) => xs:string()
+};
+
+declare
+    %test:assertEquals("<node>data</node>")
+function serialize:serialize3() {
+    let $directory := helper:get-test-directory($serialize:suite)
+    let $_ := file:mkdirs($directory)
+
+    let $path := $directory || "/serialize-3-test.txt"
+    let $_ := file:serialize($serialize:text, $path, ())
+    let $_ := file:serialize($serialize:text, $path, ())
+
+    return file:read($path)
+};
+
+declare
+    %test:assertTrue
+function serialize:xml-defaults() {
+    let $directory := helper:get-test-directory($serialize:suite)
+    let $_ := file:mkdirs($directory)
+
+    let $path := $directory || "/xml-defaults-test.xml"
+    let $_ := file:serialize($serialize:xml, $path, ())
+
+    return file:read($path) eq
+        "<root>" || $fixtures:NL ||
+        "    <unary attribute=""value""/>" || $fixtures:NL ||
+        "    <nested>" || $fixtures:NL ||
+        "        text" || $fixtures:NL ||
+        "    </nested>" || $fixtures:NL ||
+        "</root>"
+};
+
+declare
+    %test:assertTrue
+function serialize:xml-final-newline() {
+    let $directory := helper:get-test-directory($serialize:suite)
+    let $_ := file:mkdirs($directory)
+
+    let $path := $directory || "/xml-final-newline.xml"
+    let $_ := file:serialize($serialize:xml, $path,
+    <output:serialization-parameters
+            xmlns:output="http://www.w3.org/2010/xslt-xquery-serialization">
+          <exist:insert-final-newline value="yes" />
+        </output:serialization-parameters>)
+
+    return file:read($path) eq
+        "<root>" || $fixtures:NL ||
+        "    <unary attribute=""value""/>" || $fixtures:NL ||
+        "    <nested>" || $fixtures:NL ||
+        "        text" || $fixtures:NL ||
+        "    </nested>" || $fixtures:NL ||
+        "</root>" || $fixtures:NL
+};
+
+declare
+    %test:assertTrue
+function serialize:xml-minified() {
+    let $directory := helper:get-test-directory($serialize:suite)
+    let $_ := file:mkdirs($directory)
+
+    let $path := $directory || "/xml-minified.xml"
+    let $_ := file:serialize($serialize:xml, $path,
+    <output:serialization-parameters
+        xmlns:output="http://www.w3.org/2010/xslt-xquery-serialization">
+      <output:indent value="no"/>
+    </output:serialization-parameters>)
+
+    return file:read($path) eq
+        "<root><unary attribute=""value""/><nested>" || $fixtures:NL ||
+        "        text" || $fixtures:NL ||
+        "    </nested></root>"
+};
 
-    let $node-set := text {"data"}
-    let $path := system:get-exist-home() || "/test.txt"
-    let $parameters := ()
-    let $append := true()
-    let $remove := file:delete($path)
-    let $ser1 := 	file:serialize($node-set, $path, (), false())
-    let $ser2 := 	file:serialize($node-set, $path, (), false())
-    let $read := file:read($path)
-    let $remove := file:delete($path)
-    return ($read, $ser1, $ser2, $remove)
-};
diff --git a/extensions/modules/file/src/test/xquery/modules/file/sync-serialize.xqm b/extensions/modules/file/src/test/xquery/modules/file/sync-serialize.xqm
@@ -199,3 +199,51 @@ function syse:unindented-no-declaration() {
         )
     )
 };
+
+declare
+    %test:assertEquals("true", "true")
+function syse:insert-final-newline-yes() {
+    let $directory := helper:get-test-directory($syse:suite)
+    let $sync := file:sync(
+        $fixtures:collection,
+        $directory,
+        map{ "exist:insert-final-newline": true() }
+    )
+
+    return (
+        helper:assert-file-contents(
+            $fixtures:XML_DECLARATION || $fixtures:NL ||
+            $fixtures:SIMPLE_XML_INDENTED || $fixtures:NL,
+            ($directory, $syse:simple-file-name)
+        ),
+        helper:assert-file-contents(
+            $fixtures:XML_DECLARATION || $fixtures:NL ||
+            $fixtures:COMPLEX_XML_INDENTED || $fixtures:NL,
+            ($directory, $syse:complex-file-name)
+        )
+    )
+};
+
+declare
+    %test:assertEquals("true", "true")
+function syse:insert-final-newline-no() {
+    let $directory := helper:get-test-directory($syse:suite)
+    let $sync := file:sync(
+        $fixtures:collection,
+        $directory,
+        map{ "exist:insert-final-newline": false() }
+    )
+
+    return (
+        helper:assert-file-contents(
+            $fixtures:XML_DECLARATION || $fixtures:NL ||
+            $fixtures:SIMPLE_XML_INDENTED,
+            ($directory, $syse:simple-file-name)
+        ),
+        helper:assert-file-contents(
+            $fixtures:XML_DECLARATION || $fixtures:NL ||
+            $fixtures:COMPLEX_XML_INDENTED,
+            ($directory, $syse:complex-file-name)
+        )
+    )
+};
