Format test generator

Author: joefarebrother

java/ql/src/utils/GenerateFlowTestCase.py

@@ -12,7 +12,7 @@
 import tempfile
 
 if any(s == "--help" for s in sys.argv):
-  print("""Usage:
+    print("""Usage:
 GenerateFlowTestCase.py specsToTest.csv projectPom.xml outdir
 
 This generates test cases exercising function model specifications found in specsToTest.csv
@@ -27,27 +27,28 @@
 After test generation completes, any lines in specsToTest.csv that didn't produce tests are output.
 If this happens, check the spelling of class and method names, and the syntax of input and output specifications.
 """)
-  sys.exit(0)
+    sys.exit(0)
 
 if len(sys.argv) != 4:
-  print("Usage: GenerateFlowTestCase.py specsToTest.csv projectPom.xml outdir", file=sys.stderr)
-  print("specsToTest.csv should contain CSV rows describing method taint-propagation specifications to test", file=sys.stderr)
-  print("projectPom.xml should import dependencies sufficient to resolve the types used in specsToTest.csv", file=sys.stderr)
-  sys.exit(1)
+    print("Usage: GenerateFlowTestCase.py specsToTest.csv projectPom.xml outdir", file=sys.stderr)
+    print("specsToTest.csv should contain CSV rows describing method taint-propagation specifications to test", file=sys.stderr)
+    print("projectPom.xml should import dependencies sufficient to resolve the types used in specsToTest.csv", file=sys.stderr)
+    sys.exit(1)
 
 try:
-  os.makedirs(sys.argv[3])
+    os.makedirs(sys.argv[3])
 except Exception as e:
-  if e.errno != errno.EEXIST:
-    print("Failed to create output directory %s: %s" % (sys.argv[3], e))
-    sys.exit(1)
+    if e.errno != errno.EEXIST:
+        print("Failed to create output directory %s: %s" % (sys.argv[3], e))
+        sys.exit(1)
 
 resultJava = os.path.join(sys.argv[3], "Test.java")
 resultQl = os.path.join(sys.argv[3], "test.ql")
 
 if os.path.exists(resultJava) or os.path.exists(resultQl):
-  print("Won't overwrite existing files '%s' or '%s'" % (resultJava, resultQl), file = sys.stderr)
-  sys.exit(1)
+    print("Won't overwrite existing files '%s' or '%s'" %
+          (resultJava, resultQl), file=sys.stderr)
+    sys.exit(1)
 
 workDir = tempfile.mkdtemp()
 
@@ -57,129 +58,153 @@
 os.makedirs(projectDir)
 
 try:
-  shutil.copyfile(sys.argv[2], os.path.join(projectDir, "pom.xml"))
+    shutil.copyfile(sys.argv[2], os.path.join(projectDir, "pom.xml"))
 except Exception as e:
-  print("Failed to read project POM %s: %s" % (sys.argv[2], e), file = sys.stderr)
-  sys.exit(1)
+    print("Failed to read project POM %s: %s" %
+          (sys.argv[2], e), file=sys.stderr)
+    sys.exit(1)
 
 commentRegex = re.compile("^\s*(//|#)")
+
+
 def isComment(s):
-  return commentRegex.match(s) is not None
+    return commentRegex.match(s) is not None
+
 
 try:
-  with open(sys.argv[1], "r") as f:
-    specs = [l for l in f if not isComment(l)]
+    with open(sys.argv[1], "r") as f:
+        specs = [l for l in f if not isComment(l)]
 except Exception as e:
-  print("Failed to open %s: %s\n" % (sys.argv[1], e))
-  sys.exit(1)
+    print("Failed to open %s: %s\n" % (sys.argv[1], e))
+    sys.exit(1)
 
 projectTestPkgDir = os.path.join(projectDir, "src", "main", "java", "test")
 projectTestFile = os.path.join(projectTestPkgDir, "Test.java")
 
 os.makedirs(projectTestPkgDir)
 
+
 def qualifiedOuterNameFromCsvRow(row):
-  cells = row.split(";")
-  if len(cells) < 2:
-    return None
-  return cells[0] + "." + cells[1].replace("$", ".")
+    cells = row.split(";")
+    if len(cells) < 2:
+        return None
+    return cells[0] + "." + cells[1].replace("$", ".")
+
 
 with open(projectTestFile, "w") as testJava:
-  testJava.write("package test;\n\npublic class Test {\n\n")
+    testJava.write("package test;\n\npublic class Test {\n\n")
 
-  for i, spec in enumerate(specs):
-    outerName = qualifiedOuterNameFromCsvRow(spec)
-    if outerName is None:
-      print("A taint specification has the wrong format: should be 'package;classname;methodname....'", file = sys.stderr)
-      print("Mis-formatted row: " + spec, file = sys.stderr)
-      sys.exit(1)
-    testJava.write("\t%s obj%d = null;\n" % (outerName, i))
+    for i, spec in enumerate(specs):
+        outerName = qualifiedOuterNameFromCsvRow(spec)
+        if outerName is None:
+            print("A taint specification has the wrong format: should be 'package;classname;methodname....'", file=sys.stderr)
+            print("Mis-formatted row: " + spec, file=sys.stderr)
+            sys.exit(1)
+        testJava.write("\t%s obj%d = null;\n" % (outerName, i))
 
-  testJava.write("}")
+    testJava.write("}")
 
 print("Creating project database")
 cmd = ["codeql", "database", "create", "--language=java", "db"]
-ret = subprocess.call(cmd, cwd = projectDir)
+ret = subprocess.call(cmd, cwd=projectDir)
 if ret != 0:
-  print("Failed to create project database. Check that '%s' is a valid POM that pulls in all necessary dependencies, and '%s' specifies valid classes and methods." % (sys.argv[2], sys.argv[1]), file = sys.stderr)
-  print("Failed command was: %s (cwd: %s)" % (shlex.join(cmd), projectDir), file = sys.stderr)
-  sys.exit(1)
+    print("Failed to create project database. Check that '%s' is a valid POM that pulls in all necessary dependencies, and '%s' specifies valid classes and methods." % (
+        sys.argv[2], sys.argv[1]), file=sys.stderr)
+    print("Failed command was: %s (cwd: %s)" %
+          (shlex.join(cmd), projectDir), file=sys.stderr)
+    sys.exit(1)
 
 print("Creating test-generation query")
 queryDir = os.path.join(workDir, "query")
 os.makedirs(queryDir)
 qlFile = os.path.join(queryDir, "gen.ql")
 with open(os.path.join(queryDir, "qlpack.yml"), "w") as f:
-  f.write("name: test-generation-query\nversion: 0.0.0\nlibraryPathDependencies: codeql-java")
+    f.write("name: test-generation-query\nversion: 0.0.0\nlibraryPathDependencies: codeql-java")
 with open(qlFile, "w") as f:
-  f.write("import java\nimport utils.GenerateFlowTestCase\n\nclass GenRow extends TargetSummaryModelCsv {\n\n\toverride predicate row(string r) {\n\t\tr = [\n")
-  f.write(",\n".join('\t\t\t"%s"' % spec.strip() for spec in specs))
-  f.write("\n\t\t]\n\t}\n}\n")
+    f.write(
+        "import java\nimport utils.GenerateFlowTestCase\n\nclass GenRow extends TargetSummaryModelCsv {\n\n\toverride predicate row(string r) {\n\t\tr = [\n")
+    f.write(",\n".join('\t\t\t"%s"' % spec.strip() for spec in specs))
+    f.write("\n\t\t]\n\t}\n}\n")
 
 print("Generating tests")
 generatedBqrs = os.path.join(queryDir, "out.bqrs")
-cmd = ['codeql', 'query', 'run', qlFile, '--database', os.path.join(projectDir, "db"), '--output', generatedBqrs]
+cmd = ['codeql', 'query', 'run', qlFile, '--database',
+       os.path.join(projectDir, "db"), '--output', generatedBqrs]
 ret = subprocess.call(cmd)
 if ret != 0:
-  print("Failed to generate tests. Failed command was: " + shlex.join(cmd))
-  sys.exit(1)
+    print("Failed to generate tests. Failed command was: " + shlex.join(cmd))
+    sys.exit(1)
 
 generatedJson = os.path.join(queryDir, "out.json")
-cmd = ['codeql', 'bqrs', 'decode', generatedBqrs, '--format=json', '--output', generatedJson]
+cmd = ['codeql', 'bqrs', 'decode', generatedBqrs,
+       '--format=json', '--output', generatedJson]
 ret = subprocess.call(cmd)
 if ret != 0:
-  print("Failed to decode BQRS. Failed command was: " + shlex.join(cmd))
-  sys.exit(1)
-
-def getTuples(queryName, jsonResult, fname):
-  if queryName not in jsonResult or "tuples" not in jsonResult[queryName]:
-    print("Failed to read generated tests: expected key '%s' with a 'tuples' subkey in file '%s'" % (queryName, fname), file = sys.stderr)
+    print("Failed to decode BQRS. Failed command was: " + shlex.join(cmd))
     sys.exit(1)
-  return jsonResult[queryName]["tuples"]
 
-with open(generatedJson, "r") as f:
-  generateOutput = json.load(f)
-  expectedTables = ("getTestCase", "getASupportMethodModel", "missingSummaryModelCsv", "getAParseFailure")
 
-  testCaseRows, supportModelRows, missingSummaryModelCsvRows, parseFailureRows = \
-    tuple([getTuples(k, generateOutput, generatedJson) for k in expectedTables])
+def getTuples(queryName, jsonResult, fname):
+    if queryName not in jsonResult or "tuples" not in jsonResult[queryName]:
+        print("Failed to read generated tests: expected key '%s' with a 'tuples' subkey in file '%s'" % (
+            queryName, fname), file=sys.stderr)
+        sys.exit(1)
+    return jsonResult[queryName]["tuples"]
 
-  if len(testCaseRows) != 1 or len(testCaseRows[0]) != 1:
-    print("Expected exactly one getTestCase result with one column (got: %s)" % json.dumps(testCaseRows), file = sys.stderr)
-  if any(len(row) != 1 for row in supportModelRows):
-    print("Expected exactly one column in getASupportMethodModel relation (got: %s)" % json.dumps(supportModelRows), file = sys.stderr)
-  if any(len(row) != 2 for row in parseFailureRows):
-    print("Expected exactly two columns in parseFailureRows relation (got: %s)" % json.dumps(parseFailureRows), file = sys.stderr)
 
-  if len(missingSummaryModelCsvRows) != 0:
-    print("Tests for some CSV rows were requested that were not in scope (SummaryModelCsv.row does not hold):\n" + "\n".join(r[0] for r in missingSummaryModelCsvRows))
-    sys.exit(1)
-  if len(parseFailureRows) != 0:
-    print("The following rows failed to generate any test case. Check package, class and method name spelling, and argument and result specifications:\n%s" % "\n".join(r[0] + ": " + r[1] for r in parseFailureRows), file = sys.stderr)
-    sys.exit(1)
+with open(generatedJson, "r") as f:
+    generateOutput = json.load(f)
+    expectedTables = ("getTestCase", "getASupportMethodModel",
+                      "missingSummaryModelCsv", "getAParseFailure")
+
+    testCaseRows, supportModelRows, missingSummaryModelCsvRows, parseFailureRows = \
+        tuple([getTuples(k, generateOutput, generatedJson)
+               for k in expectedTables])
+
+    if len(testCaseRows) != 1 or len(testCaseRows[0]) != 1:
+        print("Expected exactly one getTestCase result with one column (got: %s)" %
+              json.dumps(testCaseRows), file=sys.stderr)
+    if any(len(row) != 1 for row in supportModelRows):
+        print("Expected exactly one column in getASupportMethodModel relation (got: %s)" %
+              json.dumps(supportModelRows), file=sys.stderr)
+    if any(len(row) != 2 for row in parseFailureRows):
+        print("Expected exactly two columns in parseFailureRows relation (got: %s)" %
+              json.dumps(parseFailureRows), file=sys.stderr)
+
+    if len(missingSummaryModelCsvRows) != 0:
+        print("Tests for some CSV rows were requested that were not in scope (SummaryModelCsv.row does not hold):\n" +
+              "\n".join(r[0] for r in missingSummaryModelCsvRows))
+        sys.exit(1)
+    if len(parseFailureRows) != 0:
+        print("The following rows failed to generate any test case. Check package, class and method name spelling, and argument and result specifications:\n%s" %
+              "\n".join(r[0] + ": " + r[1] for r in parseFailureRows), file=sys.stderr)
+        sys.exit(1)
 
 with open(resultJava, "w") as f:
-  f.write(generateOutput["getTestCase"]["tuples"][0][0])
+    f.write(generateOutput["getTestCase"]["tuples"][0][0])
 
 scriptPath = os.path.dirname(sys.argv[0])
 
+
 def copyfile(fromName, toFileHandle):
-  with open(os.path.join(scriptPath, fromName), "r") as fromFileHandle:
-    shutil.copyfileobj(fromFileHandle, toFileHandle)
+    with open(os.path.join(scriptPath, fromName), "r") as fromFileHandle:
+        shutil.copyfileobj(fromFileHandle, toFileHandle)
+
 
 with open(resultQl, "w") as f:
-  copyfile("testHeader.qlfrag", f)
-  if len(supportModelRows) != 0:
-    copyfile("testModelsHeader.qlfrag", f)
-    f.write(", ".join('"%s"' % modelSpecRow[0].strip() for modelSpecRow in supportModelRows))
-    copyfile("testModelsFooter.qlfrag", f)
-  copyfile("testFooter.qlfrag", f)
+    copyfile("testHeader.qlfrag", f)
+    if len(supportModelRows) != 0:
+        copyfile("testModelsHeader.qlfrag", f)
+        f.write(", ".join('"%s"' %
+                          modelSpecRow[0].strip() for modelSpecRow in supportModelRows))
+        copyfile("testModelsFooter.qlfrag", f)
+    copyfile("testFooter.qlfrag", f)
 
 # Make an empty .expected file, since this is an inline-exectations test
 with open(os.path.join(sys.argv[3], "test.expected"), "w"):
-  pass
+    pass
 
 cmd = ['codeql', 'query', 'format', '-qq', '-i', resultQl]
 subprocess.call(cmd)
 
-shutil.rmtree(workDir)
+shutil.rmtree(workDir)