diff --git a/exist-core/src/main/resources/org/exist/xquery/lib/xqsuite/xqsuite.xql b/exist-core/src/main/resources/org/exist/xquery/lib/xqsuite/xqsuite.xql index cd096711e88..9d0d2716670 100755 --- a/exist-core/src/main/resources/org/exist/xquery/lib/xqsuite/xqsuite.xql +++ b/exist-core/src/main/resources/org/exist/xquery/lib/xqsuite/xqsuite.xql @@ -19,7 +19,7 @@ : License along with this library; if not, write to the Free Software : Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA :) -xquery version "3.0"; +xquery version "3.1"; (:~ : XQSuite - a functional test framework based on XQuery 3.0 annotations. @@ -58,6 +58,8 @@ declare variable $test:UNKNOWN_ASSERTION := QName($test:TEST_NAMESPACE, "no-such declare variable $test:WRONG_ARG_COUNT := QName($test:TEST_NAMESPACE, "wrong-number-of-arguments"); declare variable $test:TYPE_ERROR := QName($test:TEST_NAMESPACE, "type-error"); declare variable $test:UNKNOWN_ANNOTATION_VALUE_TYPE := QName($test:TEST_NAMESPACE, "unknown-annotation-value-type"); +declare variable $test:FAILURE := QName($test:TEST_NAMESPACE, "failure"); +declare variable $test:CUSTOM_ASSERTION_FAILURE_TYPE := "custom-assertion-failure"; (:~ : Main entry point into the module. Takes a sequence of function items. @@ -86,14 +88,22 @@ declare function test:suite($functions as function(*)+) { : : @return an XML report (in xUnit format) :) -declare function test:suite($functions as function(*)+, +declare function test:suite( + $functions as function(*)+, $test-ignored-function as (function(xs:string) as empty-sequence())?, $test-started-function as (function(xs:string) as empty-sequence())?, $test-failure-function as (function(xs:string, map(xs:string, item()?), map(xs:string, item()?)) as empty-sequence())?, $test-assumption-failed-function as (function(xs:string, map(xs:string, item()?)?) as empty-sequence())?, $test-error-function as (function(xs:string, map(xs:string, item()?)?) as empty-sequence())?, - $test-finished-function as (function(xs:string) as empty-sequence())?) { + $test-finished-function as (function(xs:string) as empty-sequence())? +) { let $modules := test:distinct-modules($functions) + let $runner := + test:run-tests( + ?, ?, + $test-ignored-function, $test-started-function, $test-failure-function, + $test-assumption-failed-function, $test-error-function, $test-finished-function + ) return { @@ -106,8 +116,7 @@ declare function test:suite($functions as function(*)+, let $result := if (empty($setup) or $setup/self::ok) then let $startTime := util:system-time() - let $results := - test:function-by-annotation($modFunctions, "assert", test:run-tests(?, ?, $test-ignored-function, $test-started-function, $test-failure-function, $test-assumption-failed-function, $test-error-function, $test-finished-function)) + let $results := test:function-by-annotation($modFunctions, "assert", $runner) let $elapsed := util:system-time() - $startTime return @@ -124,12 +133,31 @@ declare function test:suite($functions as function(*)+, errors="{count($functions)}"> {$setup/string()} - return - ($result, test:call-func-with-annotation($modFunctions, "tearDown", $test-error-function))[1] + return ( + $result, + test:call-func-with-annotation($modFunctions, "tearDown", $test-error-function) + )[1] } }; +declare function test:fail ($message as xs:string, $expected as item()*, $actual as item()*) as empty-sequence() { + test:fail($message, $expected, $actual, $test:CUSTOM_ASSERTION_FAILURE_TYPE) +}; + +declare function test:fail ( + $message as xs:string, + $expected as item()*, + $actual as item()*, + $type as xs:string +) as empty-sequence() { + error($test:FAILURE, $message, map { + "expected": $expected, + "actual": $actual, + "type": $type + }) +}; + (:~ : Find functions having the given annotation and call the callback function. :) @@ -158,25 +186,29 @@ declare %private function test:distinct-modules($functions as function(*)+) as x : return upon success, an error description otherwise. : Used for setUp and tearDown. :) -declare %private function test:call-func-with-annotation($functions as function(*)+, $annot as xs:string, - $test-error-function as (function(xs:string, map(xs:string, item()?)?) as empty-sequence())?) as element()? { +declare %private function test:call-func-with-annotation( + $functions as function(*)+, + $annot as xs:string, + $test-error-function as (function(xs:string, map(xs:string, item()?)?) as empty-sequence())? +) as element()? { test:function-by-annotation($functions, $annot, function($func, $meta) { try { (, $func())[1] } catch * { - if(not(empty($test-error-function))) then - $test-error-function($annot, - map { - "code": $err:code, - "description": $err:description, - "value": $err:value, - "module": $err:module, - "line-number": $err:line-number, - "column-number": $err:column-number, - "additional": $err:additional, - "xquery-stack-trace": $exerr:xquery-stack-trace, - "java-stack-trace": $exerr:java-stack-trace - } + if (not(empty($test-error-function))) then + $test-error-function( + $annot, + map { + "code": $err:code, + "description": $err:description, + "value": $err:value, + "module": $err:module, + "line-number": $err:line-number, + "column-number": $err:column-number, + "additional": $err:additional, + "xquery-stack-trace": $exerr:xquery-stack-trace, + "java-stack-trace": $exerr:java-stack-trace + } ) else () , @@ -190,43 +222,62 @@ declare %private function test:call-func-with-annotation($functions as function( : %args() annotations found. Each %arg annotation triggers one test run : using the supplied parameters. :) -declare %private function test:run-tests($func as function(*), $meta as element(function), +declare %private function test:run-tests( + $func as function(*), + $meta as element(function), $test-ignored-function as (function(xs:string) as empty-sequence())?, $test-started-function as (function(xs:string) as empty-sequence())?, $test-failure-function as (function(xs:string, map(xs:string, item()?), map(xs:string, item()?)) as empty-sequence())?, $test-assumption-failed-function as (function(xs:string, map(xs:string, item()?)?) as empty-sequence())?, $test-error-function as (function(xs:string, map(xs:string, item()?)?) as empty-sequence())?, - $test-finished-function as (function(xs:string) as empty-sequence())?) { - if($meta/annotation[ends-with(@name, ":pending")])then - ( - if(not(empty($test-ignored-function))) then $test-ignored-function(test:get-test-name($meta)) else (), - test:print-result($meta, (), { - element pending { - $meta/annotation[ends-with(@name, ":pending")]/value ! text() - } - }) - ) + $test-finished-function as (function(xs:string) as empty-sequence())? +) { + if ($meta/annotation[ends-with(@name, ":pending")]) then + ( + if (not(empty($test-ignored-function))) then + $test-ignored-function(test:get-test-name($meta)) + else () + , + test:print-result( + $meta, + (), + { + element pending { + $meta/annotation[ends-with(@name, ":pending")]/value ! text() + } + } + ) + ) else let $failed-assumptions := test:test-assumptions($meta, $test-assumption-failed-function) return - if(not(empty($failed-assumptions)))then - test:print-result($meta, (), { - element assumptions { - for $failed-assumption in $failed-assumptions - return - element assumption { - attribute name { replace($failed-assumption/@name, "[^:]+:(.+)", "$1") }, - $failed-assumption/value/text() - } - } - }) + if (not(empty($failed-assumptions))) then + test:print-result( + $meta, + (), + { + element assumptions { + for $failed-assumption in $failed-assumptions + return + element assumption { + attribute name { replace($failed-assumption/@name, "[^:]+:(.+)", "$1") }, + $failed-assumption/value/text() + } + } + } + ) else - let $argsAnnot := $meta/annotation[matches(@name, ":args?")][not(preceding-sibling::annotation[1][matches(@name, ":args?")])] - return - if ($argsAnnot) then - $argsAnnot ! test:test($func, $meta, ., $test-started-function, $test-failure-function, $test-error-function, $test-finished-function) - else - test:test($func, $meta, (), $test-started-function, $test-failure-function, $test-error-function, $test-finished-function) + let $argsAnnot := + $meta/annotation[matches(@name, ":args?")] + [not(preceding-sibling::annotation[1][matches(@name, ":args?")])] + + let $test := test:test($func, $meta, ?, + $test-started-function, $test-failure-function, $test-error-function, $test-finished-function) + return + if ($argsAnnot) then + $argsAnnot ! $test(.) + else + $test(()) }; (:~ @@ -235,23 +286,26 @@ declare %private function test:run-tests($func as function(*), $meta as element( : @param $meta the function description : @param $test-assumption-failed-function A callback for reporting the failure of assumptions : - : @return Any assumption annotations where the asusmption did not hold true + : @return Any assumption annotations where the assumption did not hold true :) declare %private -function test:test-assumptions($meta as element(function), $test-assumption-failed-function as (function(xs:string, map(xs:string, item()?)?) as empty-sequence())?) as element(annotation)* { +function test:test-assumptions( + $meta as element(function), + $test-assumption-failed-function as (function(xs:string, map(xs:string, item()?)?) as empty-sequence())? +) as element(annotation)* { let $assumption-annotations := $meta/annotation[matches(@name, "[^:]+:assume.+")] return let $failed-assumption-annotations := $assumption-annotations ! test:test-assumption(., $test-assumption-failed-function) return ( - if(not(empty($test-assumption-failed-function))) then + if (not(empty($test-assumption-failed-function))) then $failed-assumption-annotations ! $test-assumption-failed-function( - test:get-test-name($meta), - map { - "name": ./string(@name), - "value": ./value/string() - } + test:get-test-name($meta), + map { + "name": ./string(@name), + "value": ./value/string() + } ) else () , @@ -261,10 +315,13 @@ function test:test-assumptions($meta as element(function), $test-assumption-fail declare %private -function test:test-assumption($assumption-annotation as element(annotation), $test-assumption-failed-function as (function(xs:string, map(xs:string, item()?)?) as empty-sequence())?) as element(annotation)? { - if(ends-with($assumption-annotation/@name, ":assumeInternetAccess"))then +function test:test-assumption( + $assumption-annotation as element(annotation), + $test-assumption-failed-function as (function(xs:string, map(xs:string, item()?)?) as empty-sequence())? +) as element(annotation)? { + if (ends-with($assumption-annotation/@name, ":assumeInternetAccess")) then (: check for internet access :) - try { + try { let $uri := $assumption-annotation/value/text() return (: set a timeout of 3 seconds :) @@ -274,10 +331,10 @@ function test:test-assumption($assumption-annotation as element(annotation), $te let $response := http:send-request()[1] return :) - () (: nothing failed :) - } catch * { + () (: nothing failed :) + } catch * { $assumption-annotation (: return the annotation as failed :) - } + } else() }; @@ -285,39 +342,49 @@ function test:test-assumption($assumption-annotation as element(annotation), $te : The main function for running a single test. Executes the test function : and compares the result against each assertXXX annotation. :) -declare %private function test:test($func as function(*), $meta as element(function), $firstArg as element(annotation)?, +declare %private function test:test( + $func as function(*), + $meta as element(function), + $firstArg as element(annotation)?, $test-started-function as (function(xs:string) as empty-sequence())?, $test-failure-function as (function(xs:string, map(xs:string, item()?), map(xs:string, item()?)) as empty-sequence())?, $test-error-function as (function(xs:string, map(xs:string, item()?)?) as empty-sequence())?, - $test-finished-function as (function(xs:string) as empty-sequence())?) { + $test-finished-function as (function(xs:string) as empty-sequence())? +) { let $args := test:get-run-args($firstArg) let $assertions := test:get-assertions($meta, $firstArg) let $assertError := $assertions[contains(@name, ":assertError")] return if (exists($assertions)) then ( - if(not(empty($test-started-function))) then $test-started-function(test:get-test-name($meta)) else (), + if (not(empty($test-started-function))) then + $test-started-function(test:get-test-name($meta)) + else () + , try { let $result := test:call-test($func, $meta, $args) let $assertResult := test:check-assertions($assertions, $result) return if ($assertError) then ( - if(not(empty($test-failure-function))) then - $test-failure-function(test:get-test-name($meta), - (: expected :) - map { - "error": $assertError/value/string() - }, - (: actual :) - map { - "error": map { - "value": $result - } + if (not(empty($test-failure-function))) then + $test-failure-function( + test:get-test-name($meta), + (: expected :) + map { + "error": $assertError/value/string() + }, + (: actual :) + map { + "error": map { + "value": $result } + } ) else (), - test:print-result($meta, $result, + test:print-result( + $meta, + $result, @@ -326,44 +393,75 @@ declare %private function test:test($func as function(*), $meta as element(funct ) ) else ( if ($assertResult[failure] and not(empty($test-failure-function))) then - $test-failure-function(test:get-test-name($meta), - (: expected :) - map { - "value": test:expected-strings($assertResult) - }, - (: actual :) - map { - "result": test:actual-strings($assertResult) - } + $test-failure-function( + test:get-test-name($meta), + (: expected :) + map { + "value": test:expected-strings($assertResult) + }, + (: actual :) + map { + "result": test:actual-strings($assertResult) + } ) else(), test:print-result($meta, $result, $assertResult) ) + } catch test:failure { + (: when test:fail was called read expected and actual values from $err:value :) + (: expected and actual values can be function types and need to be serialized :) + let $serialized-expected := serialize($err:value?expected, map {"method": "adaptive"}) + let $serialized-actual := serialize($err:value?actual, map {"method": "adaptive"}) + + return ( + if (not(empty($test-failure-function))) then + $test-failure-function( + test:get-test-name($meta), + (: expected :) + map { "value": $serialized-expected }, + (: actual :) + map { "result": $serialized-actual } + ) + else () + , + test:print-result( + $meta, + $serialized-actual, + + + { $serialized-actual } + + ) + ) } catch * { if ($assertError) then - if ($assertError/value and not(contains($err:code, $assertError/value/string()) - or matches($err:description, $assertError/value/string())))then + if ( + $assertError/value + and not(contains($err:code, $assertError/value/string()) + or matches($err:description, $assertError/value/string())) + ) then ( - if(not(empty($test-failure-function))) then - $test-failure-function(test:get-test-name($meta), - (: expected :) - map { - "error": $assertError/value/string() - }, - (: actual :) - map { - "error": map { - "code": $err:code, - "description": $err:description, - "value": $err:value, - "module": $err:module, - "line-number": $err:line-number, - "column-number": $err:column-number, - "additional": $err:additional, - "xquery-stack-trace": $exerr:xquery-stack-trace, - "java-stack-trace": $exerr:java-stack-trace - } + if (not(empty($test-failure-function))) then + $test-failure-function( + test:get-test-name($meta), + (: expected :) + map { + "error": $assertError/value/string() + }, + (: actual :) + map { + "error": map { + "code": $err:code, + "description": $err:description, + "value": $err:value, + "module": $err:module, + "line-number": $err:line-number, + "column-number": $err:column-number, + "additional": $err:additional, + "xquery-stack-trace": $exerr:xquery-stack-trace, + "java-stack-trace": $exerr:java-stack-trace } + } ) else () , @@ -378,30 +476,35 @@ declare %private function test:test($func as function(*), $meta as element(funct test:print-result($meta, (), ()) else ( - if(not(empty($test-error-function))) then - $test-error-function(test:get-test-name($meta), - map { - "code": $err:code, - "description": $err:description, - "value": $err:value, - "module": $err:module, - "line-number": $err:line-number, - "column-number": $err:column-number, - "additional": $err:additional, - "xquery-stack-trace": $exerr:xquery-stack-trace, - "java-stack-trace": $exerr:java-stack-trace - } + if (not(empty($test-error-function))) then + $test-error-function( + test:get-test-name($meta), + map { + "code": $err:code, + "description": $err:description, + "value": $err:value, + "module": $err:module, + "line-number": $err:line-number, + "column-number": $err:column-number, + "additional": $err:additional, + "xquery-stack-trace": $exerr:xquery-stack-trace, + "java-stack-trace": $exerr:java-stack-trace + } ) else () , - test:print-result($meta, (), + test:print-result( + $meta, + (), ) ) }, - if(not(empty($test-finished-function))) then $test-finished-function(test:get-test-name($meta)) else () + if (not(empty($test-finished-function))) then + $test-finished-function(test:get-test-name($meta)) + else () ) else () @@ -412,7 +515,9 @@ declare function test:expected-strings($report as element(report)+) { for $report-failure in $report/failure return string-join($report-failure/text(), ", ") || " (" || $report-failure/@message || ")" - , ", ") + , + ", " + ) }; declare function test:actual-strings($report as element(report)+) { @@ -479,7 +584,11 @@ declare %private function test:get-run-args($firstArg as element(annotation)?) a : Map any arguments from the %args or %arg annotations into function parameters and evaluate : the resulting function. :) -declare %private function test:call-test($func as function(*), $meta as element(function), $args as element(annotation)*) { +declare %private function test:call-test( + $func as function(*), + $meta as element(function), + $args as element(annotation)* +) { let $funArgs := if ($args[1]/@name = "test:args") then test:map-arguments($args/value, $meta/argument) @@ -710,7 +819,10 @@ declare %private function test:print-result($meta as element(function), $result (:~ : Check the function's return value against each assertion. :) -declare %private function test:check-assertions($assertions as element(annotation)*, $result as item()*) as element(report)* { +declare %private function test:check-assertions( + $assertions as element(annotation)*, + $result as item()* +) as element(report)* { for $annotation in $assertions let $assert := replace($annotation/@name, "^\w+:(.*)$", "$1") return diff --git a/exist-core/src/test/xquery/xqsuite/custom-assertion.xqm b/exist-core/src/test/xquery/xqsuite/custom-assertion.xqm new file mode 100644 index 00000000000..a83aaa32c51 --- /dev/null +++ b/exist-core/src/test/xquery/xqsuite/custom-assertion.xqm @@ -0,0 +1,165 @@ +(: + : eXist-db Open Source Native XML Database + : Copyright (C) 2001 The eXist-db Authors + : + : info@exist-db.org + : http://www.exist-db.org + : + : This library is free software; you can redistribute it and/or + : modify it under the terms of the GNU Lesser General Public + : License as published by the Free Software Foundation; either + : version 2.1 of the License, or (at your option) any later version. + : + : This library is distributed in the hope that it will be useful, + : but WITHOUT ANY WARRANTY; without even the implied warranty of + : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + : Lesser General Public License for more details. + : + : You should have received a copy of the GNU Lesser General Public + : License along with this library; if not, write to the Free Software + : Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + :) +xquery version "3.1"; + +(:~ + : Some tests on features of the test suite itself. + :) +module namespace ca="http://exist-db.org/xquery/test/xqsuite/custom-assertion"; + +import module namespace test="http://exist-db.org/xquery/xqsuite" + at "resource:org/exist/xquery/lib/xqsuite/xqsuite.xql"; + +declare variable $ca:var := map {"a": 1, "b": 2}; + +declare + %test:assertEquals("Custom message", "expected", "actual", "custom-assertion-failure") +function ca:test-fail-3() as item()* { + try { + test:fail("Custom message", "expected", "actual") + } + catch test:failure { + $err:description, $err:value?expected, $err:value?actual, $err:value?type + } +}; + +declare + %test:assertEquals("Custom message", "expected", "actual", "custom-type") +function ca:test-fail-4() as item()* { + try { + test:fail("Custom message", "expected", "actual", "custom-type") + } + catch test:failure { + $err:description, $err:value?expected, $err:value?actual, $err:value?type + } +}; + +declare + %test:assertEquals("Custom message", "expected", "actual", "custom-assertion-failure") +function ca:test-fail-fallback() as item()* { + try { + error($test:FAILURE, "Custom message", map { + "expected": "expected", + "actual": "actual", + "type": $test:CUSTOM_ASSERTION_FAILURE_TYPE + }) + } + catch test:failure { + $err:description, $err:value?expected, $err:value?actual, $err:value?type + } +}; + +declare + %test:assertTrue +function ca:map-assertion-pass() as item()* { + ca:map-assertion($ca:var, map {"b": 2, "a": 1}) +}; + +declare + %test:assertEquals("Key 'b' is missing", "{""a"":1}", "map-assertion-failure") +function ca:map-assertion-missing-key() as item()* { + try { + ca:map-assertion($ca:var, map {"a": 1}) + } + catch test:failure { + $err:description, + fn:serialize($err:value?actual, map{"method":"json"}), + $err:value?type + } +}; + +declare + %test:assertEquals("Value mismatch for key 'b'", "{""b"":3,""a"":1}", "map-assertion-failure") +function ca:map-assertion-wrong-value() as item()* { + try { + ca:map-assertion($ca:var, map {"a": 1, "b": 3}) + } + catch test:failure { + $err:description, + fn:serialize($err:value?actual, map{"method":"json"}), + $err:value?type + } +}; + +declare + %test:assertEquals("Additional keys found: (o, 23)", "{""a"":1,""o"":""o"",""23"":3}", "map-assertion-failure") +function ca:map-assertion-additional-key() as item()* { + try { + ca:map-assertion($ca:var, map {"a": 1, 23: 3, "o": "o"}) + } + catch test:failure { + $err:description, + fn:serialize($err:value?actual, map{"method":"json"}), + $err:value?type + } +}; + +declare + %test:assertEquals("Type mismatch", "[1,2]", "type-mismatch") +function ca:map-assertion-type-mismatch() as item()* { + try { + ca:map-assertion($ca:var, [1,2]) + } + catch test:failure { + $err:description, + fn:serialize($err:value?actual, map{"method":"json"}), + $err:value?type + } +}; + +(: + : custom assertion, which could also be imported from a library module + :) + +declare %private variable $ca:MAP_ASSERTION_TYPE := "map-assertion-failure"; + +declare %private +function ca:map-assertion ($expected as map(*), $actual as item()*) as item()* { + if (not(exists($actual))) + then test:fail("Actual is empty", $expected, $actual, "type-mismatch") + else if (count($actual) gt 1) + then test:fail("Actual is a sequence with more than one item", $expected, $actual, "type-mismatch") + else if (not($actual instance of map(*))) + then test:fail("Type mismatch", $expected, $actual, "type-mismatch") + else if (not(empty( + map:keys(map:remove($actual, map:keys($expected)))))) + then test:fail( + "Additional keys found: (" || string-join( + map:keys(map:remove($actual, map:keys($expected))), ', ') || ")", + $expected, + $actual, + $ca:MAP_ASSERTION_TYPE + ) + else ( + for-each(map:keys($expected), ca:map-assert-key(?, $expected, $actual)), + true() + ) +}; + +declare %private +function ca:map-assert-key ($key as xs:anyAtomicType, $expected as map(*), $actual as map(*)) as item()* { + if (not(map:contains($actual, $key))) + then test:fail("Key '" || $key || "' is missing", $expected, $actual, $ca:MAP_ASSERTION_TYPE) + else if ($expected($key) ne $actual($key)) + then test:fail("Value mismatch for key '" || $key || "'", $expected, $actual, $ca:MAP_ASSERTION_TYPE) + else () +}; diff --git a/exist-core/src/test/xquery/xqsuite/xqsuite-tests.xql b/exist-core/src/test/xquery/xqsuite/xqsuite-tests.xql index 5a90fb55f80..9ae03b138f0 100644 --- a/exist-core/src/test/xquery/xqsuite/xqsuite-tests.xql +++ b/exist-core/src/test/xquery/xqsuite/xqsuite-tests.xql @@ -19,7 +19,7 @@ : License along with this library; if not, write to the Free Software : Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA :) -xquery version "3.0"; +xquery version "3.1"; (:~ : Some tests on features of the test suite itself. @@ -28,7 +28,7 @@ module namespace t="http://exist-db.org/xquery/test/xqsuite"; declare namespace test="http://exist-db.org/xquery/xqsuite"; -declare +declare %test:assertXPath("/name[. = 'Item1']") function t:xpath() { diff --git a/exist-core/src/test/xquery/xquery3/load-xquery-module.xql b/exist-core/src/test/xquery/xquery3/load-xquery-module.xql index 4ba68cdb531..8b7c0b669fe 100644 --- a/exist-core/src/test/xquery/xquery3/load-xquery-module.xql +++ b/exist-core/src/test/xquery/xquery3/load-xquery-module.xql @@ -96,7 +96,7 @@ function lxm:import-wrong-version() { }; declare - %test:assertEquals(7) + %test:assertEquals(8) function lxm:import-from-classpath() { let $module := load-xquery-module("http://exist-db.org/xquery/xqsuite", map { "location-hints": "resource:org/exist/xquery/lib/xqsuite/xqsuite.xql"