Swift: add autobuild failure diagnostics

Author: redsun82

swift/integration-tests/create_database_utils.py

@@ -1,15 +1,34 @@
 """
-recreation of internal `create_database_utils.py` to run the tests locally, with minimal
-and swift-specialized functionality
+Simplified version of internal `create_database_utils.py` used to run the tests locally, with
+minimal and swift-specialized functionality
+TODO unify integration testing code across the public and private repositories
 """
 import subprocess
 import pathlib
 import sys
+import shutil
 
 
-def run_codeql_database_create(cmds, lang, keep_trap=True):
+def runSuccessfully(cmd):
+    res = subprocess.run(cmd)
+    if res.returncode:
+        print("FAILED", file=sys.stderr)
+        print(" ", *cmd, f"(exit code {res.returncode})", file=sys.stderr)
+        sys.exit(res.returncode)
+
+def runUnsuccessfully(cmd):
+    res = subprocess.run(cmd)
+    if res.returncode == 0:
+        print("FAILED", file=sys.stderr)
+        print(" ", *cmd, f"(exit code 0, expected to fail)", file=sys.stderr)
+        sys.exit(1)
+
+
+def run_codeql_database_create(cmds, lang, keep_trap=True, db=None, runFunction=runSuccessfully):
+    """ db parameter is here solely for compatibility with the internal test runner """
     assert lang == 'swift'
     codeql_root = pathlib.Path(__file__).parents[2]
+    shutil.rmtree("db", ignore_errors=True)
     cmd = [
         "codeql", "database", "create",
         "-s", ".", "-l", "swift", "--internal-use-lua-tracing", f"--search-path={codeql_root}", "--no-cleanup",
@@ -19,8 +38,4 @@ def run_codeql_database_create(cmds, lang, keep_trap=True):
     for c in cmds:
         cmd += ["-c", c]
     cmd.append("db")
-    res = subprocess.run(cmd)
-    if res.returncode:
-        print("FAILED", file=sys.stderr)
-        print(" ", *cmd, file=sys.stderr)
-        sys.exit(res.returncode)
+    runFunction(cmd)

swift/integration-tests/diagnostics_test_utils.py

@@ -0,0 +1,64 @@
+"""
+Simplified POSIX only version of internal `diagnostics_test_utils.py` used to run the tests locally
+TODO unify integration testing code across the public and private repositories
+"""
+
+import json
+import pathlib
+import subprocess
+import os
+import difflib
+import sys
+
+
+def _normalize_actual(test_dir, database_dir):
+    proc = subprocess.run(['codeql', 'database', 'export-diagnostics', '--format', 'raw', '--', database_dir],
+                          stdout=subprocess.PIPE, universal_newlines=True, check=True, text=True)
+    data = proc.stdout.replace(str(test_dir.absolute()), "<test-root-directory>")
+    data = json.loads(data)
+    data[:] = [e for e in data if not e["source"]["id"].startswith("cli/")]
+    for e in data:
+        e.pop("timestamp")
+    return _normalize_json(data)
+
+
+def _normalize_expected(test_dir):
+    with open(test_dir / "diagnostics.expected") as expected:
+        text = expected.read()
+    return _normalize_json(_load_concatenated_json(text))
+
+
+def _load_concatenated_json(text):
+    text = text.lstrip()
+    entries = []
+    decoder = json.JSONDecoder()
+    while text:
+        obj, index = decoder.raw_decode(text)
+        entries.append(obj)
+        text = text[index:].lstrip()
+    return entries
+
+
+def _normalize_json(data):
+    entries = [json.dumps(e, sort_keys=True, indent=2) for e in data]
+    entries.sort()
+    entries.append("")
+    return "\n".join(entries)
+
+
+def check_diagnostics(test_dir=".", test_db="db"):
+    test_dir = pathlib.Path(test_dir)
+    test_db = pathlib.Path(test_db)
+    actual = _normalize_actual(test_dir, test_db)
+    if os.environ.get("CODEQL_INTEGRATION_TEST_LEARN") == "true":
+        with open(test_dir / "diagnostics.expected", "w") as expected:
+            expected.write(actual)
+        return
+    expected = _normalize_expected(test_dir)
+    if actual != expected:
+        with open(test_dir / "diagnostics.actual", "w") as actual_out:
+            actual_out.write(actual)
+        actual = actual.splitlines(keepends=True)
+        expected = expected.splitlines(keepends=True)
+        print("".join(difflib.unified_diff(actual, expected, fromfile="diagnostics.actual", tofile="diagnostics.expected")), file=sys.stderr)
+        sys.exit(1)

swift/integration-tests/osx-only/autobuilder/failure/.gitignore

@@ -0,0 +1 @@
+/build

swift/integration-tests/osx-only/autobuilder/failure/diagnostics.expected

@@ -0,0 +1,17 @@
+{
+  "helpLinks": [
+    "https://docs.github.com/en/enterprise-server/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/customizing-code-scanning"
+  ],
+  "plaintextMessage": "The detected build command failed (tried /usr/bin/xcodebuild build -project <test-root-directory>/hello-failure.xcodeproj -target hello-failure CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO).\n\nSet up a manual build command.",
+  "severity": "error",
+  "source": {
+    "extractorName": "swift",
+    "id": "swift/autobuilder/build-command-failed",
+    "name": "Detected build command failed"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}