Skip to content

run docstrings tests with mocha #7220

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
Dec 30, 2024
Merged

run docstrings tests with mocha #7220

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
123d2b9
run tests with mocha
aspeddro Dec 28, 2024
9d5780c
trigger ci
aspeddro Dec 28, 2024
5de1563
remove ci branch
aspeddro Dec 28, 2024
856f721
test on all platform
aspeddro Dec 28, 2024
b8cf442
fix windows cli path
aspeddro Dec 28, 2024
1d014a3
fix test
aspeddro Dec 28, 2024
81c10c3
skip windows
aspeddro Dec 28, 2024
79d1346
order output
aspeddro Dec 28, 2024
877b39b
fix order
aspeddro Dec 28, 2024
5713897
cleanup
aspeddro Dec 28, 2024
af55142
adjusts
aspeddro Dec 29, 2024
adfa1a9
Remove `generated_mocha_test.res` if file exists
aspeddro Dec 29, 2024
59f1ab1
format
aspeddro Dec 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,6 @@ jobs:
# Run the slower tasks on the fastest runner only
build_playground: true
benchmarks: true
docstring_tests: true
- os: buildjet-2vcpu-ubuntu-2204-arm # ARM
ocaml_compiler: ocaml-variants.5.2.1+options,ocaml-option-static
upload_binaries: true
Expand Down Expand Up @@ -313,11 +312,6 @@ jobs:
- name: Run tests
run: node scripts/test.js -all

- name: Docstrings tests
if: matrix.docstring_tests
# Ignore functions that are not available on Node 18
run: node tests/docstrings_examples/DocTest.res.mjs --ignore-runtime-tests "Array.toReversed, Array.toSorted, Promise.withResolvers, Set.union, Set.isSupersetOf, Set.isSubsetOf, Set.isDisjointFrom, Set.intersection, Set.symmetricDifference, Set.difference"

- name: Check for diffs in tests folder
run: git diff --ignore-cr-at-eol --exit-code tests

Expand Down
30 changes: 22 additions & 8 deletions scripts/test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,14 +136,28 @@ async function runTests() {
cwd: path.join(__dirname, "..", "tests/docstrings_examples"),
stdio: [0, 1, 2],
});
// Ignore some tests not supported by node v18
// cp.execSync(
// `node ${path.join("tests", "docstrings_examples", "DocTest.res.mjs")} --ignore-runtime-tests "Array.toReversed, Array.toSorted, Promise.withResolvers, Set.union, Set.isSupersetOf, Set.isSubsetOf, Set.isDisjointFrom, Set.intersection, Set.symmetricDifference, Set.difference"`,
// {
// cwd: path.join(__dirname, ".."),
// stdio: [0, 1, 2],
// },
// );

cp.execSync(`node ${path.join("tests", "docstrings_examples", "DocTest.res.mjs")}`, {
cwd: path.join(__dirname, ".."),
stdio: [0, 1, 2],
})

cp.execSync(`${rescript_exe} build`, {
cwd: path.join(__dirname, "..", "tests/docstrings_examples"),
stdio: [0, 1, 2],
});

console.log("Formatting generated_mocha_test.res")
cp.execSync(`./cli/rescript format ${path.join("tests", "docstrings_examples", "generated_mocha_test.res")}`, {
cwd: path.join(__dirname, ".."),
stdio: [0, 1, 2],
})

console.log("Run mocha test")
cp.execSync(`npx mocha ${path.join("tests", "docstrings_examples", "generated_mocha_test.res.mjs")}`, {
cwd: path.join(__dirname, ".."),
stdio: [0, 1, 2],
});
}
}
}
Expand Down
196 changes: 47 additions & 149 deletions tests/docstrings_examples/DocTest.res
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,24 +9,19 @@ type example = {
docstrings: array<string>,
}

type error =
| ReScriptError(string)
| RuntimeError({rescript: string, js: string, error: string})

let bscBin = Path.join(["cli", "bsc"])

let parsed = Util.parseArgs({
args: Process.argv->Array.sliceToEnd(~start=2),
options: dict{"ignore-runtime-tests": {Util.type_: "string"}},
})

let ignoreRuntimeTests = switch parsed.values->Dict.get("ignore-runtime-tests") {
| Some(v) =>
v
->String.split(",")
->Array.map(s => s->String.trim)
| None => []
}
// Ignore some tests not supported by node v18
let ignoreRuntimeTests = [
Copy link
Member

@cknitt cknitt Dec 28, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Bonus points: detect node version and run tests accordingly. As these tests actually work with Node 22.

Copy link
Contributor Author

@aspeddro aspeddro Dec 29, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done.

"Array.toReversed",
"Array.toSorted",
"Promise.withResolvers",
"Set.union",
"Set.isSupersetOf",
"Set.isSubsetOf",
"Set.isDisjointFrom",
"Set.intersection",
"Set.symmetricDifference",
"Set.difference",
]

let getOutput = buffer =>
buffer
Expand Down Expand Up @@ -158,142 +153,45 @@ let extractExamples = async () => {
examples
}

let compileTest = async (~code) => {
// NOTE: warnings argument (-w) should be before eval (-e) argument
let args = ["-w", "-3-109-44", "-e", code]
let {stderr, stdout} = await SpawnAsync.run(~command=bscBin, ~args)

stderr->Array.length > 0 ? Error(stderr->getOutput) : Ok(stdout->getOutput)
}

let compileExamples = async examples => {
Console.log(`Compiling ${examples->Array.length->Int.toString} examples from docstrings...`)

let compiled = []
let compilationErrors = []

await examples->ArrayUtils.forEachAsyncInBatches(~batchSize, async example => {
// let id = example.id->String.replaceAll(".", "__")
let rescriptCode = example->getCodeBlocks

switch await compileTest(~code=rescriptCode) {
| Ok(jsCode) => compiled->Array.push((example, rescriptCode, jsCode))
| Error(err) => compilationErrors->Array.push((example, ReScriptError(err)))
}
})

(compiled, compilationErrors)
}

let runTest = async code => {
let {stdout, stderr, code: exitCode} = await SpawnAsync.run(
~command="node",
~args=["-e", code, "--input-type", "commonjs"],
~options={cwd: Process.cwd(), timeout: 2000},
let main = async () => {
let examples = await extractExamples()
examples->Array.sort((a, b) =>
Obj.magic(a.id) > Obj.magic(b.id) ? Ordering.fromInt(1) : Ordering.fromInt(-1)
Copy link
Member

@cknitt cknitt Dec 28, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we need Obj.magic here?

I think this can just be

  examples->Array.sort((a, b) => String.compare(a.id, b.id))

and should be moved into the extractExamples function.

Copy link
Contributor Author

@aspeddro aspeddro Dec 29, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks

)

// Some expressions, like `console.error("error")` are printed to stderr,
// but exit code is 0
let std = switch exitCode->Null.toOption {
| Some(exitCode) if exitCode == 0.0 && Array.length(stderr) > 0 => stderr->Ok
| Some(exitCode) if exitCode == 0.0 => stdout->Ok
| None | Some(_) => Error(Array.length(stderr) > 0 ? stderr : stdout)
}

switch std {
| Ok(buf) => Ok(buf->getOutput)
| Error(buf) => Error(buf->getOutput)
}
}

let runExamples = async compiled => {
Console.log(`Running ${compiled->Array.length->Int.toString} compiled examples...`)

let tests = compiled->Array.filter((({id}, _, _)) => !(ignoreRuntimeTests->Array.includes(id)))

let runtimeErrors = []
await tests->ArrayUtils.forEachAsyncInBatches(~batchSize, async compiled => {
let (example, rescriptCode, jsCode) = compiled

switch await runTest(jsCode) {
| Ok(_) => ()
| Error(error) =>
let runtimeError = RuntimeError({rescript: rescriptCode, js: jsCode, error})
runtimeErrors->Array.push((example, runtimeError))
let testsContent =
examples
->Array.filterMap(example => {
let codeExamples = getCodeBlocks(example)

let ignore = Array.includes(ignoreRuntimeTests, example.id)

switch String.length(codeExamples) == 0 {
| true => None
| false =>
ignore
? None
: Some(
`describe("${example.id}", () => {
test("${example.id}", () => {
module Test = {
${codeExamples}
}
()
})
})`,
)
}
})
->Array.join("\n\n")

runtimeErrors
}

let indentOutputCode = code => {
let indent = String.repeat(" ", 2)

code
->String.split("\n")
->Array.map(s => `${indent}${s}`)
->Array.join("\n")
}

let printErrors = errors => {
errors->Array.forEach(((example, errors)) => {
let red = s => `\x1B[1;31m${s}\x1B[0m`
let cyan = s => `\x1b[36m${s}\x1b[0m`
let kind = switch example.kind {
| "moduleAlias" => "module alias"
| other => other
}

let a = switch errors {
| ReScriptError(error) =>
let err =
error
->String.split("\n")
// Drop line of filename
->Array.filterWithIndex((_, i) => i !== 2)
->Array.join("\n")

`${"error"->red}: failed to compile examples from ${kind} ${example.id->cyan}
${err}`
| RuntimeError({rescript, js, error}) =>
let indent = String.repeat(" ", 2)

`${"runtime error"->red}: failed to run examples from ${kind} ${example.id->cyan}

${indent}${"ReScript"->cyan}

${rescript->indentOutputCode}

${indent}${"Compiled Js"->cyan}

${js->indentOutputCode}

${indent}${"stacktrace"->red}

${error->indentOutputCode}
`
}

Process.stderrWrite(a)
})
}

let main = async () => {
let examples = await extractExamples()
let (compiled, compilationErrors) = await compileExamples(examples)
let runtimeErrors = await runExamples(compiled)
let dirname = url->URL.fileURLToPath->Path.dirname
let filepath = Path.join([dirname, "generated_mocha_test.res"])
let fileContent = `open Mocha
@@warning("-32-34-60-37-109-3-44")

let allErrors = Array.concat(runtimeErrors, compilationErrors)
${testsContent}`

if allErrors->Array.length > 0 {
printErrors(allErrors)
1
} else {
Console.log("All examples passed successfully")
0
}
await Fs.writeFile(filepath, fileContent)
}

let exitCode = await main()

Process.exit(exitCode)
let () = await main()
Loading