Merge pull request github#11455 from michaelnebel/java/flowtestcaseextensions

Java: Update the flow test case generator to produce data extensions.

Author: michaelnebel

csharp/ql/lib/ext/generated/dotnet_runtime.model.yml

@@ -1,6 +1,5 @@
- 
 # THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT.
-# Definitions of taint steps in the dotnet_runtime framework.
+# Definitions of models for the dotnet_runtime framework.
 
 extensions:
 

java/ql/lib/ext/generated/org.apache.commons.io.model.yml

@@ -1,6 +1,5 @@
- 
 # THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT.
-# Definitions of taint steps in the org.apache.commons.io framework.
+# Definitions of models for the org.apache.commons.io framework.
 
 extensions:
   - addsTo:

java/ql/src/utils/flowtestcasegenerator/FlowTestCase.qll

@@ -12,7 +12,7 @@ private import FlowTestCaseSupportMethods
 
 /**
  * A CSV row to generate tests for. Users should extend this to define which
- * tests to generate. Rows specified here should also satisfy `SummaryModelCsv.row`.
+ * tests to generate. There should already exist summaries for the rows specified here.
  */
 class TargetSummaryModelCsv extends Unit {
   /**
@@ -42,11 +42,11 @@ predicate summaryModelRow(
 }
 
 /**
- * Gets a CSV row for which a test has been requested, but `SummaryModelCsv.row` does not hold of it.
+ * Gets a CSV row for which a test has been requested, but where a summary has not already been defined.
  */
-query string missingSummaryModelCsv() {
+query string missingSummaryModel() {
   any(TargetSummaryModelCsv target).row(result) and
-  not any(SummaryModelCsv model).row(result)
+  not summaryModelRow(_, _, _, _, _, _, _, _, _, _, result)
 }
 
 /**

java/ql/src/utils/flowtestcasegenerator/FlowTestCaseSupportMethods.qll

@@ -97,12 +97,12 @@ abstract class SupportMethod extends string {
   int getPriority() { result = 50 }
 
   /**
-   * Gets the CSV row describing this support method if it is needed to set up the output for this test.
+   * Gets the data extension row describing this support method if it is needed to set up the output for this test.
    *
-   * For example, `newWithMapValue` will propagate a value from `Argument[0]` to `MapValue of ReturnValue`, and `getMapValue`
+   * For example, `newWithMapValue` will propagate a value from `Argument[0]` to `ReturnValue.MapValue`, and `getMapValue`
    * will do the opposite.
    */
-  string getCsvModel() { none() }
+  string getDataExtensionModel() { none() }
 }
 
 /**
@@ -162,10 +162,11 @@ private class DefaultGetMethod extends GetMethod {
     result = "Object get" + contentToken(c) + "Default(Object container) { return null; }"
   }
 
-  override string getCsvModel() {
+  override string getDataExtensionModel() {
     result =
-      "generatedtest;Test;false;" + this.getName() + ";(Object);;Argument[0]." +
-        getComponentSpec(SummaryComponent::content(c)) + ";ReturnValue;value;manual"
+      "\"generatedtest\", \"Test\", False, \"" + this.getName() +
+        "\", \"(Object)\", \"\", \"Argument[0]." + getComponentSpec(SummaryComponent::content(c)) +
+        "\", \"ReturnValue\", \"value\", \"manual\""
   }
 }
 
@@ -358,10 +359,11 @@ private class DefaultGenMethod extends GenMethod {
     result = "Object newWith" + contentToken(c) + "Default(Object element) { return null; }"
   }
 
-  override string getCsvModel() {
+  override string getDataExtensionModel() {
     result =
-      "generatedtest;Test;false;" + this.getName() + ";(Object);;Argument[0];ReturnValue." +
-        getComponentSpec(SummaryComponent::content(c)) + ";value;manual"
+      "\"generatedtest\", \"Test\", False, \"" + this.getName() +
+        "\", \"(Object)\", \"\", \"Argument[0]\", \"ReturnValue." +
+        getComponentSpec(SummaryComponent::content(c)) + "\", \"value\", \"manual\""
   }
 }
 

java/ql/src/utils/flowtestcasegenerator/GenerateFlowTestCase.py

@@ -24,7 +24,9 @@
 
 If --force is present, existing files may be overwritten.
 
-Requirements: `mvn` and `codeql` should both appear on your path.
+Requirements:
+ - `mvn` and `codeql` should both appear on your path.
+ - `--additional-packs /path/to/semmle-code/ql` should be added to your `.config/codeql/config` file.
 
 After test generation completes, any lines in specsToTest.csv that didn't produce tests are output.
 If this happens, check the spelling of class and method names, and the syntax of input and output specifications.
@@ -52,10 +54,12 @@
 
 resultJava = os.path.join(sys.argv[3], "Test.java")
 resultQl = os.path.join(sys.argv[3], "test.ql")
+resultYml = os.path.join(sys.argv[3], "test.model.yml")
+resultPack = os.path.join(sys.argv[3], "qlpack.yml")
 
-if not force and (os.path.exists(resultJava) or os.path.exists(resultQl)):
-    print("Won't overwrite existing files '%s' or '%s'" %
-          (resultJava, resultQl), file=sys.stderr)
+if not force and (os.path.exists(resultJava) or os.path.exists(resultQl) or os.path.exists(resultYml) or os.path.exists(resultPack)):
+    print("Won't overwrite existing files '%s', '%s', '%s' or '%s'." %
+          (resultJava, resultQl, resultYml, resultPack), file=sys.stderr)
     sys.exit(1)
 
 workDir = tempfile.mkdtemp()
@@ -127,7 +131,13 @@ def qualifiedOuterNameFromCsvRow(row):
 os.makedirs(queryDir)
 qlFile = os.path.join(queryDir, "gen.ql")
 with open(os.path.join(queryDir, "qlpack.yml"), "w") as f:
-    f.write("name: test-generation-query\nversion: 0.0.0\nlibraryPathDependencies: codeql/java-queries")
+    f.write("""name: test-generation-query
+version: 0.0.0
+dependencies:
+  codeql/java-all: '*'
+  codeql/java-queries: '*'
+""")
+
 with open(qlFile, "w") as f:
     f.write(
         "import java\nimport utils.flowtestcasegenerator.GenerateFlowTestCase\n\nclass GenRow extends TargetSummaryModelCsv {\n\n\toverride predicate row(string r) {\n\t\tr = [\n")
@@ -163,9 +173,9 @@ def getTuples(queryName, jsonResult, fname):
 with open(generatedJson, "r") as f:
     generateOutput = json.load(f)
     expectedTables = ("getTestCase", "getASupportMethodModel",
-                      "missingSummaryModelCsv", "getAParseFailure", "noTestCaseGenerated")
+                      "missingSummaryModel", "getAParseFailure", "noTestCaseGenerated")
 
-    testCaseRows, supportModelRows, missingSummaryModelCsvRows, parseFailureRows, noTestCaseGeneratedRows = \
+    testCaseRows, supportModelRows, missingSummaryModelRows, parseFailureRows, noTestCaseGeneratedRows = \
         tuple([getTuples(k, generateOutput, generatedJson)
                for k in expectedTables])
 
@@ -182,9 +192,9 @@ def getTuples(queryName, jsonResult, fname):
         print("Expected exactly one column in noTestCaseGenerated relation (got: %s)" %
               json.dumps(noTestCaseGeneratedRows), file=sys.stderr)
 
-    if len(missingSummaryModelCsvRows) != 0:
-        print("Tests for some CSV rows were requested that were not in scope (SummaryModelCsv.row does not hold):\n" +
-              "\n".join(r[0] for r in missingSummaryModelCsvRows))
+    if len(missingSummaryModelRows) != 0:
+        print("Tests for some CSV rows were requested that were not in scope (a summary doesn't already exist):\n" +
+              "\n".join(r[0] for r in missingSummaryModelRows))
         sys.exit(1)
     if len(parseFailureRows) != 0:
         print("The following rows failed to generate any test case. Check package, class and method name spelling, and argument and result specifications:\n%s" %
@@ -207,11 +217,32 @@ def copyfile(fromName, toFileHandle):
 
 with open(resultQl, "w") as f:
     copyfile("testHeader.qlfrag", f)
-    if len(supportModelRows) != 0:
-        copyfile("testModelsHeader.qlfrag", f)
-        f.write(", ".join('"%s"' %
-                          modelSpecRow[0].strip() for modelSpecRow in supportModelRows))
-        copyfile("testModelsFooter.qlfrag", f)
+
+if len(supportModelRows) != 0:
+    # Make a test extension file
+    with open(resultYml, "w") as f:
+        models = "\n".join('      - [%s]' %
+                          modelSpecRow[0].strip() for modelSpecRow in supportModelRows)
+        dataextensions = f"""extensions:
+  - addsTo:
+      pack: codeql/java-tests
+      extensible: extSummaryModel
+    data:
+{models}
+"""
+        f.write(dataextensions)
+    # Make a qlpack file such that the extension will be picked up
+    with open(resultPack, "w") as f:
+        f.write("""name: example-test-pack
+version: 0.0.0
+extractor: java
+dependencies:
+  codeql/java-all: '*'
+  codeql/java-queries: '*'
+  codeql/java-tests: '*'
+dataExtensions:
+  - test.model.yml
+""")
 
 # Make an empty .expected file, since this is an inline-exectations test
 with open(os.path.join(sys.argv[3], "test.expected"), "w"):

java/ql/src/utils/flowtestcasegenerator/GenerateFlowTestCase.qll

@@ -13,13 +13,13 @@ private import FlowTestCaseSupportMethods
 private import FlowTestCaseUtils
 
 /**
- * Gets a CSV row for which a test has been requested, and `SummaryModelCsv.row` does hold, but
+ * Gets a CSV row for which a test has been requested, and where there exists a summary, but
  * nonetheless we can't generate a test case for it, indicating we cannot resolve either the callable
  * spec or an input or output spec.
  */
 query string getAParseFailure(string reason) {
   any(TargetSummaryModelCsv target).row(result) and
-  any(SummaryModelCsv model).row(result) and
+  summaryModelRow(_, _, _, _, _, _, _, _, _, _, result) and
   (
     not summaryModelRow(_, _, _, _, _, _, _, _, _, _, result) and
     reason = "row could not be parsed"
@@ -52,7 +52,7 @@ query string getAParseFailure(string reason) {
  */
 query string noTestCaseGenerated() {
   any(TargetSummaryModelCsv target).row(result) and
-  any(SummaryModelCsv model).row(result) and
+  summaryModelRow(_, _, _, _, _, _, _, _, _, _, result) and
   not exists(getAParseFailure(_)) and
   not exists(any(TestCase tc).getATestSnippetForRow(result))
 }
@@ -85,12 +85,12 @@ SupportMethod getASupportMethod() {
 }
 
 /**
- * Returns a CSV specification of the taint-/value-propagation behavior of a test support method (`get` or `newWith` method).
+ * Returns a data extension specification of the taint-/value-propagation behavior of a test support method (`get` or `newWith` method).
  */
-query string getASupportMethodModel() { result = getASupportMethod().getCsvModel() }
+query string getASupportMethodModel() { result = getASupportMethod().getDataExtensionModel() }
 
 /**
- * Gets a Java file body testing all requested CSV rows against whatever classes and methods they resolve against.
+ * Gets a Java file body testing all requested Models as Data rows against whatever classes and methods they resolve against.
  */
 query string getTestCase() {
   result =

java/ql/src/utils/flowtestcasegenerator/testModelsFooter.qlfrag

@@ -173,9 +173,8 @@ def makeContent(self):
         else:
             negativeSummaryAddsTo = ""
 
-        return f""" 
-# THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT.
-# Definitions of taint steps in the {self.friendlyname} framework.
+        return f"""# THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT.
+# Definitions of models for the {self.friendlyname} framework.
 
 extensions:
 {sinkAddsTo}