Improve lab testing (#673)

* Improve lab testing

Improve how labs are tested.

Change how the "examples" are used in the hint section
so that people provide full examples up to the tested index,
and *document* this (before it wasn't clear how do this).
By using this approach we will *consistently* test hints.

In the longer term, we might remove the constraint on which
hints are returned during the testing, but that requires
determining which forms cover which indexes, which is more
complex. By asking for the fields now, we make it easier
to make that change later.

This makes changes in labs to *use* the new testing mechanism
in hint examples.

This also adds the script "mass-test", which automatically
tests all labs. It does this by opening them all in the web browser,
triggering the self-test in each lab. You have to manually look
at each page, which is a little annoying. That said,
that takes very little time. Automatically opening
every page is a big improvement in terms of automated testing.

Signed-off-by: David A. Wheeler <[email protected]>

* Remove trailing space

Signed-off-by: David A. Wheeler <[email protected]>

---------

Signed-off-by: David A. Wheeler <[email protected]>

Author: david-a-wheeler

docs/labs/checker.js

@@ -425,12 +425,14 @@ function processHints(requestedHints) {
         newHint.text = hint.text;
         // Precompile all regular expressions & report any failures
         if (hint.present) {
+            newHint.present = hint.present;
             newHint.presentRe = processRegex(hint.present,
                 `hint[${i}].present`, false);
         };
         if (hint.absent) {
+            newHint.absent = hint.absent;
             newHint.absentRe = processRegex(hint.absent,
-                `hint[${i}].present`, false);
+                `hint[${i}].absent`, false);
         };
         if (!hint.absent && !hint.present && (i != requestedHints.length - 1)) {
             showDebugOutput(
@@ -565,11 +567,11 @@ function runSelftest() {
     for (let hint of hints) {
         if (hint.examples) {
             for (let example of hint.examples) {
-		// Create a testAttempt
-                let testAttempt = expected.slice(); // shallow copy of expected
-                testAttempt[hint.index] = example;
-		// What hint does our new testAttempt give?
-                actualHint = findHint(testAttempt, [hint.index]);
+		// We directly pass our example.
+		// This means that examples will need to contain multiple
+		// values if the index > 0. We only return hints with the
+		// given hint index.
+                actualHint = findHint(example, [hint.index]);
                 if (actualHint != hint.text) {
                     alert(`Lab Error: Unexpected hint!\n\nExample:\n${example}\n\nExpected hint:\n${hint.text}\n\nProduced hint:\n${actualHint}\n\nExpected (passing example)=${JSON.stringify(expected)}\n\ntestAttempt=${JSON.stringify(testAttempt)}\nFailing hint=${JSON.stringify(hint)}`);
                 };

docs/labs/create_checker.md

@@ -364,9 +364,17 @@ If you want to check an index other than `0`, add an `index` field and provide
 an integer.
 
 A hint can include an `examples` field, which must then contain
-an array of examples (each example is an array of Strings).
+an array of examples which are used as tests.
+Each example is an array of Strings; each element
+corresponds to the indexes.
 On load the system will verify that each example will report the
-maatching hint (this helps ensure that the hint order is sensible).
+matching hint (this helps ensure that the hint order is sensible).
+
+At the time of this writing, all examples are loaded and used as tests
+to ensure that the hint requested is actually the one reported.
+If your example is for a later index, provide test values that
+don't trigger earlier index values. Currently those values are ignored,
+but future versions will probably use them when checking the examples.
 
 #### Examples of hints
 
@@ -398,6 +406,21 @@ the hint.
 The second hint triggers when the user attempt *contains* the given
 pattern (note the term `present`).
 
+The "examples" shown here are for a common case: the index is 0.
+Once you have multiple index, you'll need to use a longer form for
+examples with larger indexes:
+
+~~~~yaml
+  examples:
+  -
+    - "  VALUE FOR INDEX0"
+    - "  VALUE FOR INDEX1"
+  -
+    - "  VALUE FOR INDEX0"
+    - "  VALUE FOR INDEX1"
+~~~~yaml
+
+
 ### Notes on YAML
 
 The info section supports
@@ -648,7 +671,7 @@ Here is an example:
 
 ~~~~yaml
 preprocessing:
-  - 
+  -
     - |-
         [\n\r]+
     - ""

docs/labs/deserialization.html

@@ -115,10 +115,10 @@
   text: Begin the second section with `if ( data.username && ... `
     because you must check if data is even present before you can check
     various attributes of that data.
-#   examples:
-#   -
-#     - "const data = JSON.parse(base64Decoded);"
-#     - "if (typeof data.username == 'string' && data.username.length < 20 && data.username) {"
+  examples:
+  -
+    - "const data = JSON.parse(base64Decoded);"
+    - "if (typeof data.username == 'string' && data.username.length < 20 && data.username) {"
 successes:
   -
     - const data = JSON.parse(base64Decoded);

docs/labs/handling-errors.html

@@ -79,45 +79,67 @@
   present: "{ (.*?)} "
   text: Try simply returning the result of the division.
   examples:
-  - - "  return { success: true, result: a / b };"
-  - - "  return { result: a / b };"
+  -
+    - throw new Error("Division by zero is not allowed");
+    - "  return { success: true, result: a / b };"
+  -
+    - throw new Error("Division by zero is not allowed");
+    - "  return { result: a / b };"
 - index: 2
   absent: '\s*try\s*{\s* '
   text: >-
     Use a try block to catch any exceptions that might be thrown.
     It should look something like `try { ... } catch(err) {...}`
     (fill in the `...` sections).
   examples:
-  - - "  const result = divide(10, 2);"
+  -
+    - throw new Error("Division by zero is not allowed");
+    - return a / b;
+    - "  const result = divide(10, 2);"
 - index: 2
   present: '\s* try \s* { .*? if \( result.success \) .*?'
   text: You may assume that the result is successful within the try block.
   examples:
-  - - " try { const result = divide(10 ,2); if( result.success) { console.log ( \"Result:\", result ); "
+  -
+    - throw new Error("Division by zero is not allowed");
+    - return a / b;
+    - " try { const result = divide(10 ,2); if( result.success) { console.log ( \"Result:\", result ); "
 - index: 2
   present: '.*? result.result .*?'
   text: The result is not an object, it is a number.
   examples:
-  - - " try { const result = divide(10 ,2); console.log ( \"Result:\", result.result ); "
+  -
+    - throw new Error("Division by zero is not allowed");
+    - return a / b;
+    - " try { const result = divide(10 ,2); console.log ( \"Result:\", result.result ); "
 - index: 2
   absent: '.*? catch .*? '
   text: >-
     Handle the error within the catch block. You need `catch(err) {...}`
     after `try {...}` to catch an error in the try block.
   examples:
-  - - " try { const result = divide(10 ,2); console.log ( \"Result:\", result ); }"
+  -
+    - throw new Error("Division by zero is not allowed");
+    - return a / b;
+    - " try { const result = divide(10 ,2); console.log ( \"Result:\", result ); }"
 - index: 2
   absent: '\s* catch \s* \( .*? \) { \s* '
   text: Use 'catch (...) {...}' to catch an error object within the catch block.
   examples:
-  - - " try { const result = divide(10 ,2); console.log ( \"Result:\", result ); } catch {}"
+  -
+    - throw new Error("Division by zero is not allowed");
+    - return a / b;
+    - " try { const result = divide(10 ,2); console.log ( \"Result:\", result ); } catch {}"
 - index: 2
   absent: |-
     catch \( err \)
   text: >-
     Please use `catch(err) {...}` for purposes of this lab.
   examples:
-  - - " try { const result = divide(10 ,2); console.log ( \"Result:\", result ); } catch (foo) {"
+  -
+    - throw new Error("Division by zero is not allowed");
+    - return a / b;
+    - " try { const result = divide(10 ,2); console.log ( \"Result:\", result ); } catch (foo) {"
 - index: 2
   present: |-
     catch .* console \. error \( ["'][^"']*["'] , result
@@ -129,8 +151,14 @@
     the variable `result` is out of scope in the catch block anyway;
     it was declared in the try block.
   examples:
-  - - " try { const result = divide(10 ,2); console.log ( \"Result:\", result ); } catch (err) { console.error('Error', result.message);"
-  - - " try { const result = divide(10 ,2); console.log ( \"Result:\", result ); } catch (err) { console.error('Error', result );"
+  -
+    - throw new Error("Division by zero is not allowed");
+    - return a / b;
+    - " try { const result = divide(10 ,2); console.log ( \"Result:\", result ); } catch (err) { console.error('Error', result.message);"
+  -
+    - throw new Error("Division by zero is not allowed");
+    - return a / b;
+    - " try { const result = divide(10 ,2); console.log ( \"Result:\", result ); } catch (err) { console.error('Error', result );"
 # debug: true
 </script>
 </head>

docs/labs/mass-test

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+# Mass open all lab files. Do this by opening every lab in a web browser,
+# which will invoke each lab's built-in tests.
+
+# Create a list of labs
+grep -o '[A-Za-z0-9_-]*\.html' README.md | sort |uniq > ,1
+
+OPENER=xdg-open
+if ! which "$OPENER" >/dev/null; then
+  OPENER=open
+fi
+
+for file in $(cat ,1); do
+  ${OPENER} "$file"
+done
+
+echo 'Check each lab file to ensure there are no error alerts and that'
+echo 'there is a yellow field for input.'

docs/labs/regex1.html

@@ -139,26 +139,26 @@
     ^\^
   index: 1
   text: For input validation, start with '^' to indicate a full match.
-  # examples:
-  # -
-  #   - "^[YN]$"
-  #   - ""
+  examples:
+  -
+    - "^[YN]$"
+    - ""
 - absent: |-
     \$$
   index: 1
   text: For input validation, end with '$' to indicate a full match.
-  #   examples:
-  #   -
-  #     - "^[YN]$"
-  #     - "^"
+  examples:
+  -
+    - "^[YN]$"
+    - "^"
 - absent: |-
     \[A-Z\]
   index: 1
   text: You can use [A-Z] to match one uppercase Latin letter (A through Z).
-  # examples:
-  # -
-  #   - "^[YN]$"
-  #   - "^$"
+  examples:
+  -
+    - "^[YN]$"
+    - "^$"
 - present: |-
     \^\[A-Z\]\*
   index: 1
@@ -172,10 +172,10 @@
      \[A-Z\]\[A-Z\]\*)
   index: 1
   text: You can use [A-Z]+ to match one or more uppercase Latin letters.
-  # examples:
-  # -
-  #   - "^[YN]$"
-  #   - "^[A-Z]$"
+  examples:
+  -
+    - "^[YN]$"
+    - "^[A-Z]$"
 - present: "True"
   index: 2
   text: Regular expressions are case-sensitive by default; use "true".