Test case generator: improve error reporting

We now distinguish cases where SSV rows are not in scope at all from those where they don't identify a known type or method, or where input or output specs could not be parsed.

Author: smowton

java/ql/src/utils/GenerateFlowTestCase.py

@@ -15,7 +15,7 @@
   print("""Usage:
 GenerateFlowTestCase.py specsToTest.ssv projectPom.xml outdir
 
-This generates test cases exercising taint flow specifications found in specsToTest.ssv
+This generates test cases exercising function model specifications found in specsToTest.ssv
 producing files Test.java, test.ql and test.expected in outdir.
 
 projectPom.xml should be a Maven pom sufficient to resolve the classes named in specsToTest.ssv.
@@ -115,11 +115,6 @@ def qualifiedOuterNameFromSsvRow(row):
   f.write("import java\nimport utils.GenerateFlowTestCase\n\nclass GenRow extends CsvRow {\n\n\tGenRow() {\n\t\tthis = [\n")
   f.write(",\n".join('\t\t\t"%s"' % spec.strip() for spec in specs))
   f.write("\n\t\t]\n\t}\n}\n")
-  f.write("""
-query string getAFailedRow() {
-  result = any(GenRow row | not exists(RowTestSnippet r | exists(r.getATestSnippetForRow(row))) | row)
-}
-  """)
 
 print("Generating tests")
 generatedBqrs = os.path.join(queryDir, "out.bqrs")
@@ -144,17 +139,24 @@ def getTuples(queryName, jsonResult, fname):
 
 with open(generatedJson, "r") as f:
   generateOutput = json.load(f)
-  testCaseRows = getTuples("getTestCase", generateOutput, generatedJson)
-  supportModelRows = getTuples("getASupportMethodModel", generateOutput, generatedJson)
-  failedRows = getTuples("getAFailedRow", generateOutput, generatedJson)
+  expectedTables = ("getTestCase", "getASupportMethodModel", "missingSummaryModelCsv", "getAParseFailure")
+
+  testCaseRows, supportModelRows, missingSummaryModelCsvRows, parseFailureRows = \
+    tuple([getTuples(k, generateOutput, generatedJson) for k in expectedTables])
+
   if len(testCaseRows) != 1 or len(testCaseRows[0]) != 1:
     print("Expected exactly one getTestCase result with one column (got: %s)" % json.dumps(testCaseRows), file = sys.stderr)
   if any(len(row) != 1 for row in supportModelRows):
     print("Expected exactly one column in getASupportMethodModel relation (got: %s)" % json.dumps(supportModelRows), file = sys.stderr)
-  if any(len(row) != 1 for row in failedRows):
-    print("Expected exactly one column in getAFailedRow relation (got: %s)" % json.dumps(failedRows), file = sys.stderr)
-  if len(failedRows) != 0:
-    print("The following rows failed to generate any test case. Check package, class and method name spelling, and argument and result specifications:\n%s" % "\n".join(r[0] for r in failedRows), file = sys.stderr)
+  if any(len(row) != 2 for row in parseFailureRows):
+    print("Expected exactly two columns in parseFailureRows relation (got: %s)" % json.dumps(parseFailureRows), file = sys.stderr)
+
+  if len(missingSummaryModelCsvRows) != 0:
+    print("Tests for some SSV rows were requested that were not in scope (SummaryModelCsv.row does not hold):\n" + "\n".join(r[0] for r in missingSummaryModelCsvRows))
+    sys.exit(1)
+  if len(parseFailureRows) != 0:
+    print("The following rows failed to generate any test case. Check package, class and method name spelling, and argument and result specifications:\n%s" % "\n".join(r[0] + ": " + r[1] for r in parseFailureRows), file = sys.stderr)
+    sys.exit(1)
 
 with open(resultJava, "w") as f:
   f.write(generateOutput["getTestCase"]["tuples"][0][0])

java/ql/src/utils/GenerateFlowTestCase.qll

@@ -5,10 +5,48 @@ import semmle.code.java.dataflow.FlowSummary
 import semmle.code.java.dataflow.internal.FlowSummaryImpl
 
 /**
- * A CSV row to generate tests for. Users should extend this to define their input rows.
+ * A CSV row to generate tests for. Users should extend this to define which
+ * tests to generate. Rows specified here should also satisfy `SummaryModelCsv.row`.
  */
 bindingset[this]
-abstract class CsvRow extends string { }
+abstract class TargetSummaryModelCsv extends string {
+  predicate modelRowExists() { any(SummaryModelCsv smc).row(this) }
+}
+
+/**
+ * Gets a CSV row for which a test has been requested, but `SummaryModelCsv.row` does not hold of it.
+ */
+query TargetSummaryModelCsv missingSummaryModelCsv() { not result.modelRowExists() }
+
+/**
+ * Gets a CSV row for which a test has been requested, and `SummaryModelCsv.row` does hold, but
+ * nonetheless we can't generate a test case for it, indicating we cannot resolve either the callable
+ * spec or an input or output spec.
+ */
+query TargetSummaryModelCsv getAParseFailure(string reason) {
+  result.modelRowExists() and
+  (
+    exists(
+      string namespace, string type, boolean subtypes, string name, string signature, string ext
+    |
+      summaryModel(namespace, type, subtypes, name, signature, ext, _, _, _, result) and
+      not exists(interpretElement(namespace, type, subtypes, name, signature, ext)) and
+      reason = "callable could not be resolved"
+    )
+    or
+    exists(string inputSpec |
+      summaryModel(_, _, _, _, _, _, inputSpec, _, _, result) and
+      not Private::External::interpretSpec(inputSpec, _) and
+      reason = "input spec could not be parsed"
+    )
+    or
+    exists(string outputSpec |
+      summaryModel(_, _, _, _, _, _, _, outputSpec, _, result) and
+      not Private::External::interpretSpec(outputSpec, _) and
+      reason = "output spec could not be parsed"
+    )
+  )
+}
 
 /**
  * Returns type of parameter `i` of `callable`, including the type of `this` for parameter -1.