Merge pull request #431 from ossf/preprocessing_test

Labs: Add mechanism for testing preprocessor

Author: david-a-wheeler

docs/labs/checker.js

@@ -90,12 +90,11 @@ function showDebugOutput(debugOutput, alwaysAlert = true) {
 
 /**
  * Given take a regex string, preprocess it (using our array of
- * preprocessing regexes), and return a final compiled regex.
+ * preprocessing regexes), and return a processed regex as a String.
  * @regexString - String to be converted into a compiled Regex
- * @description - Description of @regexString's purpose (for error reports)
  * @fullMatch - require full match (insert "^" at beginning, "$" at end).
  */
-function processRegex(regexString, description, fullMatch = true) {
+function processRegexToString(regexString, fullMatch = true) {
     let processedRegexString = regexString;
     for (preprocessRegex of preprocessRegexes) {
         processedRegexString = processedRegexString.replace(
@@ -107,6 +106,18 @@ function processRegex(regexString, description, fullMatch = true) {
         // work correctly and the first capturing (...) will be the first.
         processedRegexString = '^(?:' + processedRegexString + ')$';
     }
+    return processedRegexString;
+}
+
+/**
+ * Given take a regex string, preprocess it (using our array of
+ * preprocessing regexes), and return a final compiled Regexp.
+ * @regexString - String to be converted into a compiled Regexp
+ * @description - Description of @regexString's purpose (for error reports)
+ * @fullMatch - require full match (insert "^" at beginning, "$" at end).
+ */
+function processRegex(regexString, description, fullMatch = true) {
+    let processedRegexString = processRegexToString(regexString, fullMatch);
     try {
         let compiledRegex = new RegExp(processedRegexString);
         return compiledRegex;
@@ -118,6 +129,18 @@ function processRegex(regexString, description, fullMatch = true) {
     }
 }
 
+/*
+ * Determine if preprocessing produces the expected final regex answer.
+ * @example - 2-element array. LHS is to be processed, RHS is expected result
+ */
+function validProcessing(example) {
+    let [unProcessed, expectedProcessed] = example;
+    let actualProcessed = processRegexToString(unProcessed, false);
+    // alert(`actual\n${actualProcessed}\nexpected\n${expectedProcessed}`);
+    // alert(`actual length\n${actualProcessed.length}\nexpected length\n${expectedProcessed.length}`);
+    return (actualProcessed == expectedProcessed);
+}
+
 /**
  * Return true iff the indexed attempt matches the indexed correct.
  * @attempt - Array of strings that might be correct
@@ -304,7 +327,7 @@ function processInfo(configurationInfo) {
 
     const allowedInfoFields = new Set([
         'hints', 'successes', 'failures', 'correct', 'expected',
-        'debug']);
+         'preprocessing', 'preprocessingTests', 'debug']);
     let usedFields = new Set(Object.keys(info));
     let forbiddenFields = usedFields.difference(allowedInfoFields);
     if (forbiddenFields.size != 0) {
@@ -364,6 +387,13 @@ function runSelftest() {
        }
     };
 
+    // Run tests of the preprocessing process, if present
+    for (let example of (info.preprocessingTests || [])) {
+        if (!validProcessing(example)) {
+            alert(`Lab Error: preprocessing\n${example.join("\n\n")}\nshould pass but fails.`);
+	    };
+    };
+
     // Run tests in successes and failures, if present
     if (info.successes) {
         for (let example of info.successes) {

docs/labs/create_checker.md

@@ -556,6 +556,17 @@ Each array element should itself be an array of:
    these buffer boundary constructs are not in ECMAScript/JavaScript,
    though there [is a proposal to add them](https://github.com/tc39/proposal-regexp-buffer-boundaries).
 
+You can also test preprocessing by setting the info field
+`preprocessingTests` - if you don't set `preprocessing` itself, you're
+testing the default preprocessor.
+The `preprocessingTests` field contains an array of examples that
+test the preprocessor.
+Each example array is two elements long;
+the first is a pattern that could be
+requested, and the second is post-processed pattern that should result.
+There's no need for a "failure" test suite here, because we
+demand exact results for every test case.
+
 Here is an example:
 
 ~~~~yaml
@@ -572,6 +583,17 @@ preprocessing:
     - |-
         (\\s\*)?[ \t]+(\\s\*)?
     - "\\s*"
+preprocessingTests:
+  -
+    - |-
+        \s* console \. log \( (["'`])Hello,\x20world!\1 \) ; \s*
+    - |-
+        \s*console\s*\.\s*log\s*\(\s*(["'`])Hello,\x20world!\1\s*\)\s*;\s*
+  -
+    - |-
+        \s* foo \s+ bar \\string\\ \s*
+    - |-
+        \s*foo\s+bar\s*\\string\\\s*
 ~~~~
 
 Here is an explanation of each of these preprocessing elements
@@ -585,3 +607,6 @@ in this example:
 3. 1+ spaces/tabs are instead interpreted as `\s*` (0+ whitespace).
    The optional expressions before and after it are an optimization,
    to coalesce this for speed.
+
+If you load `hello.html` you'll automatically run some self-tests on
+the default preprocessor.

docs/labs/hello.html

@@ -93,6 +93,19 @@
 #     - |-
 #         (\\s\*)?\s+(\\s\*)?
 #     - "\\s*"
+# Here are tests for default preprocessing. You do not need this in
+# every lab, but it demonstrates how to do it.
+preprocessingTests:
+  -
+    - |-
+        \s* console \. log \( (["'`])Hello,\x20world!\1 \) ; \s*
+    - |-
+        \s*console\s*\.\s*log\s*\(\s*(["'`])Hello,\x20world!\1\s*\)\s*;\s*
+  -
+    - |-
+        \s* foo \s+ bar \\string\\ \s*
+    - |-
+        \s*foo\s+bar\s*\\string\\\s*
 # debug: true
 </script>
 <!--