From f16ec85cbf1f8be49b3fef5af30bfcf7e54afdee Mon Sep 17 00:00:00 2001 From: "David A. Wheeler" Date: Thu, 17 Oct 2024 16:52:38 -0400 Subject: [PATCH 1/2] Improve lab testing Improve how labs are tested. Change how the "examples" are used in the hint section so that people provide full examples up to the tested index, and *document* this (before it wasn't clear how do this). By using this approach we will *consistently* test hints. In the longer term, we might remove the constraint on which hints are returned during the testing, but that requires determining which forms cover which indexes, which is more complex. By asking for the fields now, we make it easier to make that change later. This makes changes in labs to *use* the new testing mechanism in hint examples. This also adds the script "mass-test", which automatically tests all labs. It does this by opening them all in the web browser, triggering the self-test in each lab. You have to manually look at each page, which is a little annoying. That said, that takes very little time. Automatically opening every page is a big improvement in terms of automated testing. Signed-off-by: David A. Wheeler --- docs/labs/checker.js | 14 +++++----- docs/labs/create_checker.md | 27 +++++++++++++++++-- docs/labs/deserialization.html | 8 +++--- docs/labs/handling-errors.html | 48 +++++++++++++++++++++++++++------- docs/labs/mass-test | 19 ++++++++++++++ docs/labs/regex1.html | 32 +++++++++++------------ 6 files changed, 110 insertions(+), 38 deletions(-) create mode 100755 docs/labs/mass-test diff --git a/docs/labs/checker.js b/docs/labs/checker.js index a1bc0824..08fda8ac 100644 --- a/docs/labs/checker.js +++ b/docs/labs/checker.js @@ -425,12 +425,14 @@ function processHints(requestedHints) { newHint.text = hint.text; // Precompile all regular expressions & report any failures if (hint.present) { + newHint.present = hint.present; newHint.presentRe = processRegex(hint.present, `hint[${i}].present`, false); }; if (hint.absent) { + newHint.absent = hint.absent; newHint.absentRe = processRegex(hint.absent, - `hint[${i}].present`, false); + `hint[${i}].absent`, false); }; if (!hint.absent && !hint.present && (i != requestedHints.length - 1)) { showDebugOutput( @@ -565,11 +567,11 @@ function runSelftest() { for (let hint of hints) { if (hint.examples) { for (let example of hint.examples) { - // Create a testAttempt - let testAttempt = expected.slice(); // shallow copy of expected - testAttempt[hint.index] = example; - // What hint does our new testAttempt give? - actualHint = findHint(testAttempt, [hint.index]); + // We directly pass our example. + // This means that examples will need to contain multiple + // values if the index > 0. We only return hints with the + // given hint index. + actualHint = findHint(example, [hint.index]); if (actualHint != hint.text) { alert(`Lab Error: Unexpected hint!\n\nExample:\n${example}\n\nExpected hint:\n${hint.text}\n\nProduced hint:\n${actualHint}\n\nExpected (passing example)=${JSON.stringify(expected)}\n\ntestAttempt=${JSON.stringify(testAttempt)}\nFailing hint=${JSON.stringify(hint)}`); }; diff --git a/docs/labs/create_checker.md b/docs/labs/create_checker.md index f44d9312..1e407f2c 100644 --- a/docs/labs/create_checker.md +++ b/docs/labs/create_checker.md @@ -364,9 +364,17 @@ If you want to check an index other than `0`, add an `index` field and provide an integer. A hint can include an `examples` field, which must then contain -an array of examples (each example is an array of Strings). +an array of examples which are used as tests. +Each example is an array of Strings; each element +corresponds to the indexes. On load the system will verify that each example will report the -maatching hint (this helps ensure that the hint order is sensible). +matching hint (this helps ensure that the hint order is sensible). + +At the time of this writing, all examples are loaded and used as tests +to ensure that the hint requested is actually the one reported. +If your example is for a later index, provide test values that +don't trigger earlier index values. Currently those values are ignored, +but future versions will probably use them when checking the examples. #### Examples of hints @@ -398,6 +406,21 @@ the hint. The second hint triggers when the user attempt *contains* the given pattern (note the term `present`). +The "examples" shown here are for a common case: the index is 0. +Once you have multiple index, you'll need to use a longer form for +examples with larger indexes: + +~~~~yaml + examples: + - + - " VALUE FOR INDEX0" + - " VALUE FOR INDEX1" + - + - " VALUE FOR INDEX0" + - " VALUE FOR INDEX1" +~~~~yaml + + ### Notes on YAML The info section supports diff --git a/docs/labs/deserialization.html b/docs/labs/deserialization.html index dcc300a8..012f89db 100644 --- a/docs/labs/deserialization.html +++ b/docs/labs/deserialization.html @@ -115,10 +115,10 @@ text: Begin the second section with `if ( data.username && ... ` because you must check if data is even present before you can check various attributes of that data. -# examples: -# - -# - "const data = JSON.parse(base64Decoded);" -# - "if (typeof data.username == 'string' && data.username.length < 20 && data.username) {" + examples: + - + - "const data = JSON.parse(base64Decoded);" + - "if (typeof data.username == 'string' && data.username.length < 20 && data.username) {" successes: - - const data = JSON.parse(base64Decoded); diff --git a/docs/labs/handling-errors.html b/docs/labs/handling-errors.html index 6a628d83..c61f3360 100644 --- a/docs/labs/handling-errors.html +++ b/docs/labs/handling-errors.html @@ -79,8 +79,12 @@ present: "{ (.*?)} " text: Try simply returning the result of the division. examples: - - - " return { success: true, result: a / b };" - - - " return { result: a / b };" + - + - throw new Error("Division by zero is not allowed"); + - " return { success: true, result: a / b };" + - + - throw new Error("Division by zero is not allowed"); + - " return { result: a / b };" - index: 2 absent: '\s*try\s*{\s* ' text: >- @@ -88,36 +92,54 @@ It should look something like `try { ... } catch(err) {...}` (fill in the `...` sections). examples: - - - " const result = divide(10, 2);" + - + - throw new Error("Division by zero is not allowed"); + - return a / b; + - " const result = divide(10, 2);" - index: 2 present: '\s* try \s* { .*? if \( result.success \) .*?' text: You may assume that the result is successful within the try block. examples: - - - " try { const result = divide(10 ,2); if( result.success) { console.log ( \"Result:\", result ); " + - + - throw new Error("Division by zero is not allowed"); + - return a / b; + - " try { const result = divide(10 ,2); if( result.success) { console.log ( \"Result:\", result ); " - index: 2 present: '.*? result.result .*?' text: The result is not an object, it is a number. examples: - - - " try { const result = divide(10 ,2); console.log ( \"Result:\", result.result ); " + - + - throw new Error("Division by zero is not allowed"); + - return a / b; + - " try { const result = divide(10 ,2); console.log ( \"Result:\", result.result ); " - index: 2 absent: '.*? catch .*? ' text: >- Handle the error within the catch block. You need `catch(err) {...}` after `try {...}` to catch an error in the try block. examples: - - - " try { const result = divide(10 ,2); console.log ( \"Result:\", result ); }" + - + - throw new Error("Division by zero is not allowed"); + - return a / b; + - " try { const result = divide(10 ,2); console.log ( \"Result:\", result ); }" - index: 2 absent: '\s* catch \s* \( .*? \) { \s* ' text: Use 'catch (...) {...}' to catch an error object within the catch block. examples: - - - " try { const result = divide(10 ,2); console.log ( \"Result:\", result ); } catch {}" + - + - throw new Error("Division by zero is not allowed"); + - return a / b; + - " try { const result = divide(10 ,2); console.log ( \"Result:\", result ); } catch {}" - index: 2 absent: |- catch \( err \) text: >- Please use `catch(err) {...}` for purposes of this lab. examples: - - - " try { const result = divide(10 ,2); console.log ( \"Result:\", result ); } catch (foo) {" + - + - throw new Error("Division by zero is not allowed"); + - return a / b; + - " try { const result = divide(10 ,2); console.log ( \"Result:\", result ); } catch (foo) {" - index: 2 present: |- catch .* console \. error \( ["'][^"']*["'] , result @@ -129,8 +151,14 @@ the variable `result` is out of scope in the catch block anyway; it was declared in the try block. examples: - - - " try { const result = divide(10 ,2); console.log ( \"Result:\", result ); } catch (err) { console.error('Error', result.message);" - - - " try { const result = divide(10 ,2); console.log ( \"Result:\", result ); } catch (err) { console.error('Error', result );" + - + - throw new Error("Division by zero is not allowed"); + - return a / b; + - " try { const result = divide(10 ,2); console.log ( \"Result:\", result ); } catch (err) { console.error('Error', result.message);" + - + - throw new Error("Division by zero is not allowed"); + - return a / b; + - " try { const result = divide(10 ,2); console.log ( \"Result:\", result ); } catch (err) { console.error('Error', result );" # debug: true diff --git a/docs/labs/mass-test b/docs/labs/mass-test new file mode 100755 index 00000000..39937a55 --- /dev/null +++ b/docs/labs/mass-test @@ -0,0 +1,19 @@ +#!/bin/sh + +# Mass open all lab files. Do this by opening every lab in a web browser, +# which will invoke each lab's built-in tests. + +# Create a list of labs +grep -o '[A-Za-z0-9_-]*\.html' README.md | sort |uniq > ,1 + +OPENER=xdg-open +if ! which "$OPENER" >/dev/null; then + OPENER=open +fi + +for file in $(cat ,1); do + ${OPENER} "$file" +done + +echo 'Check each lab file to ensure there are no error alerts and that' +echo 'there is a yellow field for input.' diff --git a/docs/labs/regex1.html b/docs/labs/regex1.html index d7c017c4..2dd25440 100644 --- a/docs/labs/regex1.html +++ b/docs/labs/regex1.html @@ -139,26 +139,26 @@ ^\^ index: 1 text: For input validation, start with '^' to indicate a full match. - # examples: - # - - # - "^[YN]$" - # - "" + examples: + - + - "^[YN]$" + - "" - absent: |- \$$ index: 1 text: For input validation, end with '$' to indicate a full match. - # examples: - # - - # - "^[YN]$" - # - "^" + examples: + - + - "^[YN]$" + - "^" - absent: |- \[A-Z\] index: 1 text: You can use [A-Z] to match one uppercase Latin letter (A through Z). - # examples: - # - - # - "^[YN]$" - # - "^$" + examples: + - + - "^[YN]$" + - "^$" - present: |- \^\[A-Z\]\* index: 1 @@ -172,10 +172,10 @@ \[A-Z\]\[A-Z\]\*) index: 1 text: You can use [A-Z]+ to match one or more uppercase Latin letters. - # examples: - # - - # - "^[YN]$" - # - "^[A-Z]$" + examples: + - + - "^[YN]$" + - "^[A-Z]$" - present: "True" index: 2 text: Regular expressions are case-sensitive by default; use "true". From 5e423681c8d18fd8a2bb8ea6b3f96bb1aab8773c Mon Sep 17 00:00:00 2001 From: "David A. Wheeler" Date: Thu, 17 Oct 2024 17:52:55 -0400 Subject: [PATCH 2/2] Remove trailing space Signed-off-by: David A. Wheeler --- docs/labs/create_checker.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/labs/create_checker.md b/docs/labs/create_checker.md index 1e407f2c..44efad66 100644 --- a/docs/labs/create_checker.md +++ b/docs/labs/create_checker.md @@ -371,7 +371,7 @@ On load the system will verify that each example will report the matching hint (this helps ensure that the hint order is sensible). At the time of this writing, all examples are loaded and used as tests -to ensure that the hint requested is actually the one reported. +to ensure that the hint requested is actually the one reported. If your example is for a later index, provide test values that don't trigger earlier index values. Currently those values are ignored, but future versions will probably use them when checking the examples. @@ -671,7 +671,7 @@ Here is an example: ~~~~yaml preprocessing: - - + - - |- [\n\r]+ - ""