Merge pull request #194 from marklogic-community/feature/554-docs

DEVEXP-554 Added docs for debugging tests

Author: rjrudin

docs/assertion-functions.md

@@ -1,7 +1,7 @@
 ---
 layout: default
 title: Assertion functions
-nav_order: 6
+nav_order: 7
 permalink: /docs/assertions
 ---
 

docs/debugging-tests.md

@@ -0,0 +1,89 @@
+---
+layout: default
+title: Debugging tests
+nav_order: 5
+permalink: /docs/debugging
+---
+
+Like all unit testing frameworks, marklogic-unit-test is intended to speed up the cycle of developing, testing, and 
+fixing code. A critical aspect of that is understanding why a test failed and how to fix it. This page provides 
+guidance on how you can write your tests to ensure they can be easily debugged by any member of your development 
+team. 
+
+## Development setup
+
+Before looking at how to write tests, you should ensure that you can change your application code and test the 
+changes as quickly as possible. As mentioned in the [Getting started guide](/docs), ml-gradle supports 
+[watching for module changes](https://github.com/marklogic/ml-gradle/wiki/Watching-for-module-changes) so that when 
+you modify either an application module file or test module file, the file will be immediately loaded into your 
+application's modules database. This allows you to test your changes as quickly as possible.
+
+To enable this, simply run the following Gradle task in its own terminal window:
+
+    ./gradlew -i mlWatch
+
+The Gradle "-i" flag - for info-level logging - results in each modified file being logged when it is loaded into 
+MarkLogic. 
+
+## Test assertion messages
+
+Each [assertion function](/docs/assertions) in marklogic-unit-test supports an assertion message as its final argument. 
+These are recommended for when the intent of an assertion is not readily apparent from the two values being compared. 
+
+For example, consider the following assertion:
+
+    const actual = someLib.something();
+    test.assertEqual(3, actual);
+
+If that assertion fails because "actual" has a value of 2, marklogic-unit-test will throw the following error:
+
+    expected: 3 actual: 2 (ASSERT-EQUAL-FAILED)
+
+However, this doesn't explain why "3" is the expected value. The optional 3rd argument in `test.assertEqual` 
+provides a test developer with a chance to explain why the value is expected - e.g.:
+
+    const actual = someLib.something();
+    test.assertEqual(3, actual, "Due to such-and-such reason, 3 should be returned");
+
+The "such-and-such reason" would of course be replaced with a meaningful explanation in your test. 
+marklogic-unit-test will then include this message in the failure message:
+
+    Due to such-and-such reason, 3 should be returned; expected: 3 actual: 2 (ASSERT-EQUAL-FAILED)
+
+## Using log statements
+
+While tools such as [mlxprs](https://github.com/marklogic/mlxprs) exist to leverage the debugging capabilities within
+MarkLogic, you may often find it helpful to include log statements both in your application module and in your test 
+module. These can log the value of certain variables that are not returned by the function being tested but whose 
+values provide insight into how the application code is behaving. 
+
+For JavaScript code, use the following:
+
+    const value = someLib.something();
+    console.log("The value", value);
+
+And for XQuery code, use the following:
+
+    let $value := someLib:something()
+    let $_ := xdmp:log(("The value", $value))
+
+You can add these statements to your tests as well. 
+
+Log messages will then appear in the MarkLogic log file named "PORT_ErrorLog.txt", where "PORT" is the port number 
+of the MarkLogic app server that you are running the tests against.
+
+## Examining the Gradle test report
+
+When running tests via Gradle - either via `./gradlew test` or `./gradlew mlUnitTest` - one or more test failures will 
+result in the task failing with the location of the test report being logged. The test report allows you to see details
+on each test, including the failed assertion message and stacktrace for each failed test. 
+
+When running `./gradlew test`, the test report will be available at "build/reports/tests/tests/index.html". You can 
+open this file in a web browser to see the results; it is recommended to keep that window open and simply refresh it 
+after re-running the Gradle `test` task. 
+
+When running `./gradlew mlUnitTest`, the test report is a set of JUnit-formatted XML files available at 
+"build/test-results/marklogic-unit-test". This approach does not result in an HTML web page that can be viewed in a web
+browser, but each XML file will contain the results for the tests in a particular suite, including the failed assertion 
+messages and stacktrace for each failed test.
+

docs/loading-test-data.md

@@ -1,7 +1,7 @@
 ---
 layout: default
 title: Loading test data
-nav_order: 5
+nav_order: 6
 permalink: /docs/loading
 ---
 

marklogic-unit-test-client/src/main/java/com/marklogic/test/unit/JaxpServiceResponseUnmarshaller.java

@@ -15,6 +15,9 @@
 import javax.xml.transform.TransformerFactory;
 import javax.xml.transform.dom.DOMSource;
 import javax.xml.transform.stream.StreamResult;
+import java.io.ByteArrayOutputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
 import java.io.StringReader;
 import java.io.StringWriter;
 import java.util.ArrayList;
@@ -28,6 +31,7 @@ public class JaxpServiceResponseUnmarshaller implements ServiceResponseUnmarshal
 
     protected final Logger logger = LoggerFactory.getLogger(getClass());
     private DocumentBuilder documentBuilder;
+    private TransformerFactory transformerFactory;
     private static int ELEMENT_TYPE = 1;
 
     @Override
@@ -91,14 +95,23 @@ public TestSuiteResult parseTestSuiteResult(String xml) {
 
     @Override
     public JUnitTestSuite parseJUnitTestSuiteResult(String xml) {
-        Element root = parse(xml).getDocumentElement();
+        Document doc = parse(xml);
+        Element root = doc.getDocumentElement();
         int errors = Integer.parseInt(root.getAttribute("errors"));
         int failures = Integer.parseInt(root.getAttribute("failures"));
         String hostname = root.getAttribute("hostname");
         String name = root.getAttribute("name");
         int tests = Integer.parseInt(root.getAttribute("tests"));
         double time = Double.parseDouble(root.getAttribute("time"));
-        JUnitTestSuite suite = new JUnitTestSuite(xml, errors, failures, hostname, name, tests, time);
+
+        String prettyXml = xml;
+        try {
+            prettyXml = prettyPrintXml(doc);
+        } catch (Exception ex) {
+            logger.warn("Unable to pretty-print XML; cause: " + ex.getMessage());
+        }
+
+        JUnitTestSuite suite = new JUnitTestSuite(prettyXml, errors, failures, hostname, name, tests, time);
 
         NodeList testCases = root.getChildNodes();
         for (int i = 0; i < testCases.getLength(); i++) {
@@ -124,8 +137,10 @@ public JUnitTestSuite parseJUnitTestSuiteResult(String xml) {
 
     protected String toXml(Node node) {
         try {
-            TransformerFactory transFactory = TransformerFactory.newInstance();
-            Transformer transformer = transFactory.newTransformer();
+            if (transformerFactory == null) {
+                transformerFactory = TransformerFactory.newInstance();
+            }
+            Transformer transformer = transformerFactory.newTransformer();
             StringWriter buffer = new StringWriter();
             transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
             transformer.transform(new DOMSource(node), new StreamResult(buffer));
@@ -156,4 +171,25 @@ protected Document parse(String xml) {
         }
     }
 
+    /**
+     * Added this in 1.4.0 to handle pretty-printing the entire XML document, while {@code toXml} is only for the
+     * embedded failure XML. Did not want to try reusing that for the whole XML document.
+     *
+     * @param doc
+     * @return
+     * @throws Exception
+     */
+    private String prettyPrintXml(Document doc) throws Exception {
+        if (transformerFactory == null) {
+            transformerFactory = TransformerFactory.newInstance();
+        }
+        Transformer transformer = transformerFactory.newTransformer();
+        transformer.setOutputProperty(OutputKeys.METHOD, "xml");
+        transformer.setOutputProperty(OutputKeys.INDENT, "yes");
+        transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2");
+
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        transformer.transform(new DOMSource(doc), new StreamResult(new OutputStreamWriter(baos)));
+        return new String(baos.toByteArray());
+    }
 }

marklogic-unit-test-client/src/test/java/com/marklogic/test/unit/SelectTestsToRunTest.java

@@ -7,6 +7,7 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class SelectTestsToRunTest {
 
@@ -15,6 +16,12 @@ void selectTestsInSuite() {
         JUnitTestSuite suite = new TestManager(ClientUtil.getClient())
             .runSuite("Assertions", new TestManager.RunParameters("assert-all-exist.xqy", "assert-equal.xqy"));
 
+        // Simple check to verify that the XML is pretty-printed for easier manual inspection.
+        final String xml = suite.getXml();
+        final String message = "Expecting each testcase to be indented with 2 spaces; actual XML: " + xml;
+        assertTrue(xml.contains("  <testcase classname=\"assert-all-exist.xqy\""), message);
+        assertTrue(xml.contains("  <testcase classname=\"assert-equal.xqy\""), message);
+
         assertEquals("Assertions", suite.getName());
         assertEquals(2, suite.getTestCases().size());
         assertFalse(suite.hasTestFailures());