From f316826eeeeb72eda7acd1077b9e581741c3613f Mon Sep 17 00:00:00 2001 From: jangko Date: Sun, 23 May 2021 19:04:23 +0700 Subject: [PATCH 1/2] add mocha test report generator - test_validation - test_schemaintros --- .github/workflows/ci.yml | 16 +++ .gitignore | 1 + tests/mocha.nim | 276 ++++++++++++++++++++++++++++++++++++ tests/test_schemaintros.nim | 22 ++- tests/test_validation.nim | 42 ++++-- 5 files changed, 344 insertions(+), 13 deletions(-) create mode 100644 tests/mocha.nim diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8463afa..4dfd9e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,3 +173,19 @@ jobs: # toml_serialization is a test suite dependency nimble install -y toml_serialization env TEST_LANG="${{ matrix.target.TEST_LANG }}" nimble test + + - name: Run mochawesome-report-generator + if: runner.os == 'linux' && matrix.target.cpu == 'amd64' && matrix.target.TEST_LANG == 'c' && github.event_name == 'push' + run: | + cd nim-graphql + TITLE="nim-graphql" + SHA="${GITHUB_SHA::7}" + npm install --save-dev mochawesome-report-generator + ./node_modules/.bin/marge -o report -t "$TITLE-$SHA" -p "$TITLE test report" -f index.html report/validation.json + + - name: Deploy mochawesome report + if: runner.os == 'linux' && matrix.target.cpu == 'amd64' && matrix.target.TEST_LANG == 'c' && github.event_name == 'push' + uses: peaceiris/actions-gh-pages@v3 + with: + personal_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./nim-graphql/report diff --git a/.gitignore b/.gitignore index 07bf18b..257bab6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /nimcache +/report # Executables shall be put in an ignored build/ directory /build diff --git a/tests/mocha.nim b/tests/mocha.nim new file mode 100644 index 0000000..f911b69 --- /dev/null +++ b/tests/mocha.nim @@ -0,0 +1,276 @@ +import + std/[times, strutils, options, os], + std/[math, sequtils, random], + json_serialization, + json_serialization/std/options as jsopt + +type + UUID = distinct string + Tnull = distinct int + + State* {.pure.} = enum + failed = "failed" + passed = "passed" + skipped = "skipped" + pending = "pending" + + Speed = enum + fast = "fast" + slow = "slow" + + ErrObj = object + message: Option[string] + estack : Option[string] + diff : Option[string] + + TestRef* = ref Test + Test = object + title : string + fullTitle : string + timedOut : bool + duration : int64 + state : State + speed : Speed + pass : bool + fail : bool + pending : bool + context : Tnull + code : string + err : ErrObj + uuid : UUID + parentUUID: UUID + isHook : bool + skipped : bool + + Empty = object + SuiteRef = ref Suite + Suite = object + uuid : UUID + title : string + fullFile : string + file : string + beforeHooks: seq[Empty] + afterHooks : seq[Empty] + tests : seq[TestRef] + suites : seq[SuiteRef] + passes : seq[UUID] + failures : seq[UUID] + pending : seq[UUID] + skipped : seq[UUID] + duration : int64 + root : bool + rootEmpty : bool + timeout {.serializedFieldName("_timeout").} : int + + Statistic = object + suites : int + tests : int + passes : int + pending : int + failures : int + startTime {.serializedFieldName("start").} : DateTime + endTime {.serializedFieldName("end").} : DateTime + duration : int64 + testsRegistered: int + passPercent : int + pendingPercent : int + other : int + hasOther : bool + skipped : int + hasSkipped : bool + + Mocha* = object + stats : Statistic + results: seq[SuiteRef] + +const pattern = "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx" + +proc generateUUID(): UUID = + # taken from https://github.com/yglukhov/nim-only-uuid + var d = toInt(epochTime() * 100000) + proc fn(c : char): string = + var r = toInt(toFloat(d) + rand(1.0) * 16) %% 16 + d = toInt(floor(toFloat(d) / 16)) + toHex(if c == 'x': r else: r and 0x3 or 0x8, 1) + toLowerAscii(join(pattern.mapIt(if it == 'x' or it == 'y': fn(it) else: $it))).UUID + +proc `$`(u: UUID): string = + u.string + +proc newSuite*(title, fullFile: string): SuiteRef = + let (_, name) = fullFile.splitPath() + var suite = SuiteRef( + uuid : generateUUID(), + title : title, + fullFile : fullFile, + file : name, + timeout : 2000 + ) + suite + +proc add*(s, suite: SuiteRef) = + s.suites.add suite + s.duration += suite.duration + +proc add*(s: SuiteRef, t: TestRef) = + t.parentUUID = s.uuid + s.tests.add t + s.duration += t.duration + if t.pass: + s.passes.add t.uuid + if t.fail: + s.failures.add t.uuid + if t.skipped: + s.skipped.add t.uuid + +proc newTest*(title, fullTitle: string): TestRef = + var t = TestRef( + title : title, + fullTitle : fullTitle, + uuid : generateUUID(), + duration : getTime().toWinTime() + ) + t + +proc setCode*(t: TestRef, code: string) = + t.code = code + +template calcDuration() = + t.duration = (getTime().toWinTime() - t.duration) div 1_000_000_000_000_000_00'i64 + +proc setPass*(t: TestRef) = + calcDuration() + t.state = State.passed + t.pass = true + +proc setSkip*(t: TestRef) = + calcDuration() + t.state = State.skipped + t.skipped = true + +proc setFail*(t: TestRef, msg: string, estack = "", diff = "") = + calcDuration() + t.state = State.failed + t.fail = true + t.err.message = some(msg) + if estack.len > 0: + t.err.estack = some(estack) + if diff.len > 0: + t.err.diff = some(diff) + +proc setExpect*(t: TestRef, msg: string, expected, actual: string) = + calcDuration() + if expected != actual: + t.state = State.failed + t.fail = true + t.err.message = some(msg) + t.err.diff = some("- $1\n+ $2\n" % [actual, expected]) + else: + t.state = State.passed + t.pass = true + +proc setExpect*(t: TestRef, msg: string, expected, actual: int) = + calcDuration() + if expected != actual: + t.state = State.failed + t.fail = true + t.err.message = some(msg) + t.err.diff = some("- $1\n+ $2\n" % [$actual, $expected]) + else: + t.state = State.passed + t.pass = true + +proc setExpectErr*(t: TestRef, msg: string, expected, actual: string) = + calcDuration() + if expected != actual: + t.state = State.failed + t.fail = true + t.err.message = some(msg) + t.err.diff = some("- $1\n+ $2\n" % [actual, expected]) + else: + t.state = State.passed + t.pass = true + t.err.estack = some(actual) + +proc init*(_: type Mocha): Mocha = + result.stats.startTime = now().utc + +proc add*(m: var Mocha, suite: SuiteRef) = + m.results.add suite + +proc numTests(s: SuiteRef): int = + for x in s.suites: + result += x.numTests() + result += s.tests.len + +proc totalDuration(s: SuiteRef): int64 = + for x in s.suites: + result += x.totalDuration() + result += s.duration + +proc numPass(s: SuiteRef): int = + for x in s.suites: + result += x.numPass() + for n in s.tests: + if n.pass: + inc result + +proc numFail(s: SuiteRef): int = + for x in s.suites: + result += x.numFail() + for n in s.tests: + if n.fail: + inc result + +proc numSkip(s: SuiteRef): int = + for x in s.suites: + result += x.numSkip() + for n in s.tests: + if n.skipped: + inc result + +proc writeValue(writer: var JsonWriter, value: TNull) = + writer.writeValue JsonString("null") + +proc writeValue(writer: var JsonWriter, value: UUID) = + writer.writeValue $value + +proc writeValue(writer: var JsonWriter, value: Empty) = + writer.writeValue "" + +proc writeValue(writer: var JsonWriter, value: DateTime) = + writer.writeValue $value + +proc writeValue(writer: var JsonWriter, value: State) = + writer.writeValue $value + +proc writeValue(writer: var JsonWriter, value: Speed) = + writer.writeValue $value + +proc finalize*(m: var Mocha, fileName: string) = + m.stats.endTime = now().utc + m.stats.suites = m.results.len + for x in m.results: + m.stats.tests += x.numTests() + m.stats.duration += x.totalDuration() + m.stats.passes += x.numPass() + m.stats.failures += x.numFail() + m.stats.skipped += x.numSkip() + + m.stats.testsRegistered = m.stats.tests + m.stats.hasSkipped = m.stats.skipped > 0 + m.stats.passPercent = int((float(m.stats.passes) / float(m.stats.tests)) * 100.0) + + var s = SuiteRef( + uuid: generateUUID(), + root: true, + rootEmpty: true, + timeout: 2000 + ) + s.suites = system.move(m.results) + m.results = @[s] + + let (folder, _) = fileName.splitPath() + if folder.len > 0: + createDir(folder) + Json.saveFile(fileName, m) diff --git a/tests/test_schemaintros.nim b/tests/test_schemaintros.nim index 50037c7..f0bfdb2 100644 --- a/tests/test_schemaintros.nim +++ b/tests/test_schemaintros.nim @@ -9,18 +9,19 @@ import std/[os, strutils, unittest, json], - ../graphql + ../graphql, ./mocha const schemaFolder = "tests" / "schemas" introsFolder = "tests" / "execution" -proc runValidator(ctx: GraphqlRef, fileName: string, testStatusIMPL: var TestStatus) = +proc runValidator(ctx: GraphqlRef, fileName: string, testStatusIMPL: var TestStatus, mtest: TestRef) = var res = ctx.parseSchemaFromFile(fileName) check res.isOk if res.isErr: debugEcho "error when parsing: ", fileName debugEcho res.error + mtest.setFail($res.error) return const queryFile = introsFolder / "introspectionQuery.ql" @@ -29,6 +30,7 @@ proc runValidator(ctx: GraphqlRef, fileName: string, testStatusIMPL: var TestSta if res.isErr: debugEcho "error when parsing: ", queryFile debugEcho res.error + mtest.setFail($res.error) return const queryName = "IntrospectionQuery" @@ -38,16 +40,23 @@ proc runValidator(ctx: GraphqlRef, fileName: string, testStatusIMPL: var TestSta if res.isErr: debugEcho "error when executing: ", queryName debugEcho res.error + mtest.setFail($res.error) return try: let jsonRes = parseJson(resp.getString()) check ($jsonRes).len != 0 - except: + except Exception as e: check false + mtest.setFail(e.msg) + return + + mtest.setPass() proc main() = + var mocha = Mocha.init() suite "schema introspection validation": + var msuite = newSuite("schema introspection validation", currentSourcePath()) var ctx = new(GraphqlRef) let savePoint = ctx.getNameCounter() for fileName in walkDirRec(schemaFolder): @@ -56,9 +65,14 @@ proc main() = let parts = splitFile(fileName) test parts.name: - ctx.runValidator(fileName, testStatusIMPL) + var mtest = newTest(parts.name, fileName) + ctx.runValidator(fileName, testStatusIMPL, mtest) ctx.purgeSchema(true, true, true) ctx.purgeQueries(true) ctx.purgeNames(savePoint) + msuite.add mtest + + mocha.add msuite + mocha.finalize("report" / "schemaintros.json") main() diff --git a/tests/test_validation.nim b/tests/test_validation.nim index 4e1e7f8..9974c7c 100644 --- a/tests/test_validation.nim +++ b/tests/test_validation.nim @@ -12,7 +12,7 @@ import toml_serialization, ../graphql, ../graphql/test_config, ../graphql/graphql as context, - ../graphql/validator + ../graphql/validator, ./mocha type Unit = object @@ -122,48 +122,62 @@ proc convertCases(output: string) = for fileName in walkDirRec(caseFolder): ctx.runConverter(savePoint, fileName, output) -proc runValidator(ctx: GraphqlRef, unit: Unit, testStatusIMPL: var TestStatus) = +proc runValidator(ctx: GraphqlRef, unit: Unit, testStatusIMPL: var TestStatus, mtest: TestRef) = setupParser(ctx, unit) if parser.error != errNone: check (parser.error != errNone) == (unit.error.len > 0) - check $parser.err == unit.error + let parserErr = $parser.err + check parserErr == unit.error + mtest.setExpectErr("unexpected parser error", unit.error, parserErr) return ctx.validateAll(doc.root) if ctx.errKind != ErrNone: check (ctx.errKind != ErrNone) == (unit.error.len > 0) check ctx.errors.len == 1 - check $ctx.errors[0] == unit.error + let ctxErr = $ctx.errors[0] + check ctxErr == unit.error + mtest.setExpectErr("unexpected validation error", unit.error, ctxErr) return check (unit.error.len == 0) + mtest.setExpect("unexpected num error", unit.error.len, 0) -proc runSuite(ctx: GraphqlRef, savePoint: NameCounter, fileName: string, counter: var Counter) = +proc runSuite(ctx: GraphqlRef, savePoint: NameCounter, fileName: string, counter: var Counter, mocha: var Mocha) = let parts = splitFile(fileName) let cases = Toml.loadFile(fileName, TestCase) suite parts.name: + var msuite = newSuite(parts.name, fileName) for unit in cases.units: test unit.name: + var mtest = newTest(unit.name, parts.name) + mtest.setCode(unit.code) if unit.skip: skip() inc counter.skip + mtest.setSkip() else: - ctx.runValidator(unit, testStatusIMPL) + ctx.runValidator(unit, testStatusIMPL, mtest) ctx.purgeQueries(false) ctx.purgeSchema(false, false, false) ctx.purgeNames(savePoint) if testStatusIMPL == OK: inc counter.ok + mtest.setPass() else: inc counter.fail + msuite.add mtest + mocha.add msuite proc validateCases() = var counter: Counter var ctx = setupContext() let savePoint = ctx.getNameCounter() + var mocha = Mocha.init() for fileName in walkDirRec(caseFolder): - ctx.runSuite(savePoint, fileName, counter) + ctx.runSuite(savePoint, fileName, counter, mocha) debugEcho counter + mocha.finalize("report" / "validation.json") proc runValidator(ctx: GraphqlRef, fileName: string, testStatusIMPL: var TestStatus) = var stream = memFileInput(fileName) @@ -214,20 +228,30 @@ when isMainModule: var ctx = setupContext() var savePoint = ctx.getNameCounter() let fileName = caseFolder / conf.testFile + var mocha = Mocha.init() + if conf.unit.len == 0: - ctx.runSuite(savePoint, fileName, counter) + ctx.runSuite(savePoint, fileName, counter, mocha) echo counter + mocha.finalize("report" / "validation.json") return let cases = Toml.loadFile(fileName, TestCase) + var msuite = newSuite(fileName.splitFile().name, fileName) for unit in cases.units: if unit.name != conf.unit: continue test unit.name: - ctx.runValidator(unit, testStatusIMPL) + var mtest = newTest(unit.name, fileName) + ctx.runValidator(unit, testStatusIMPL, mtest) + if testStatusIMPL == OK: + mtest.setPass() ctx.purgeQueries(true) ctx.purgeSchema(true, true, true) ctx.purgeNames(savePoint) + msuite.add mtest + mocha.add msuite + mocha.finalize("report" / "validation.json") var message: string ## Processing command line arguments From 9ea810d20e005205f2ab156f9f48971a175db10b Mon Sep 17 00:00:00 2001 From: jangko Date: Sun, 23 May 2021 19:50:13 +0700 Subject: [PATCH 2/2] add test report summary link to readme.md https://status-im.github.io/nim-graphql --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index f652f0e..0ad5da8 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,8 @@ nim-graphql [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) ![Github action](https://github.com/status-im/nim-graphql/workflows/nim-graphql%20CI/badge.svg) +View test summary [here](https://status-im.github.io/nim-graphql/). + ## Introduction Enjoy writing graphql service in plain Nim!.