Improvements to the test generator:

- Only reference public methods
- Report rows for which test cases could not be generated
- Add a blanket `throws Exception` clause to the generated method

Author: joefarebrother

java/ql/src/utils/GenerateFlowTestCase.py

@@ -155,9 +155,9 @@ def getTuples(queryName, jsonResult, fname):
 with open(generatedJson, "r") as f:
     generateOutput = json.load(f)
     expectedTables = ("getTestCase", "getASupportMethodModel",
-                      "missingSummaryModelCsv", "getAParseFailure")
+                      "missingSummaryModelCsv", "getAParseFailure", "noTestCaseGenerated")
 
-    testCaseRows, supportModelRows, missingSummaryModelCsvRows, parseFailureRows = \
+    testCaseRows, supportModelRows, missingSummaryModelCsvRows, parseFailureRows, noTestCaseGeneratedRows = \
         tuple([getTuples(k, generateOutput, generatedJson)
                for k in expectedTables])
 
@@ -170,6 +170,9 @@ def getTuples(queryName, jsonResult, fname):
     if any(len(row) != 2 for row in parseFailureRows):
         print("Expected exactly two columns in parseFailureRows relation (got: %s)" %
               json.dumps(parseFailureRows), file=sys.stderr)
+    if any(len(row) != 1 for row in noTestCaseGeneratedRows):
+        print("Expected exactly one column in noTestCaseGenerated relation (got: %s)" %
+              json.dumps(noTestCaseGeneratedRows), file=sys.stderr)
 
     if len(missingSummaryModelCsvRows) != 0:
         print("Tests for some CSV rows were requested that were not in scope (SummaryModelCsv.row does not hold):\n" +
@@ -179,6 +182,9 @@ def getTuples(queryName, jsonResult, fname):
         print("The following rows failed to generate any test case. Check package, class and method name spelling, and argument and result specifications:\n%s" %
               "\n".join(r[0] + ": " + r[1] for r in parseFailureRows), file=sys.stderr)
         sys.exit(1)
+    if len(noTestCaseGeneratedRows) != 0:
+        print("The following CSV rows failed to generate any test case due to a limitation of the query. Other test cases will still be generated:\n" +
+              "\n".join(r[0] for r in noTestCaseGeneratedRows))
 
 with open(resultJava, "w") as f:
     f.write(generateOutput["getTestCase"]["tuples"][0][0])

java/ql/src/utils/GenerateFlowTestCase.qll

@@ -59,13 +59,26 @@ query string getAParseFailure(string reason) {
   )
 }
 
+/**
+ * Gets a CSV row for which a test was requested and was correctly parsed,
+ * but for which no test case could be generated due to a limitation of the query.
+ */
+query string noTestCaseGenerated() {
+  any(TargetSummaryModelCsv target).row(result) and
+  any(SummaryModelCsv model).row(result) and
+  not exists(getAParseFailure(_)) and
+  not exists(any(TestCase tc).getATestSnippetForRow(result))
+}
+
 private class CallableToTest extends Callable {
   CallableToTest() {
     exists(
       string namespace, string type, boolean subtypes, string name, string signature, string ext
     |
       summaryModel(namespace, type, subtypes, name, signature, ext, _, _, _) and
-      this = interpretElement(namespace, type, subtypes, name, signature, ext)
+      this = interpretElement(namespace, type, subtypes, name, signature, ext) and
+      this.isPublic() and
+      getRootType(this.getDeclaringType()).isPublic()
     )
   }
 }
@@ -530,7 +543,7 @@ query string getTestCase() {
   result =
     "package generatedtest;\n\n" + concat(getAnImportStatement() + "\n") +
       "\n// Test case generated by GenerateFlowTestCase.ql\npublic class Test {\n\n" +
-      concat("\t" + getASupportMethod() + "\n") + "\n\tpublic void test() {\n\n" +
+      concat("\t" + getASupportMethod() + "\n") + "\n\tpublic void test() throws Exception {\n\n" +
       concat(string row, string snippet |
         snippet = any(TestCase tc).getATestSnippetForRow(row)
       |