Update CVE report and remove unnecessary package (#692)

- Removed `libdw-dev` package from Docker image, as this is only
required for the build process.
- Added nicely formatted CVE summary output to vulnerability scan
workflow.

Author: mattkjames7

.github/workflows/reusable_cve_scan.yml

@@ -59,7 +59,7 @@ jobs:
         run: |
           python3 -m venv "${{ env.CVE_DIR }}/env" 
           source "${{ env.CVE_DIR }}/env/bin/activate"
-          pip install requests tqdm
+          pip install requests tqdm rich
           
       - name: Install Trivy
         if: ${{ inputs.run_trivy }}
@@ -191,6 +191,19 @@ jobs:
           name: ${{ inputs.image_type }}-${{ inputs.arch }}-cve-bin-tool-apt-summary
           path: "${{ env.CVE_DIR }}/cve-bin-tool-apt-summary.json"
 
+      - name: Combine Reports
+        if: ${{ inputs.run_trivy || inputs.run_grype }}
+        run: |
+          source "${{ env.CVE_DIR }}/env/bin/activate"
+          python3 scripts/combine_reports.py "${{ env.CVE_DIR }}/grype-summary.json" "${{ env.CVE_DIR }}/trivy-summary.json" 
+
+      - name: Upload Combined Report Artifact
+        if: ${{ inputs.run_trivy || inputs.run_grype }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ inputs.image_type }}-${{ inputs.arch }}-combined-report
+          path: "${{ env.CVE_DIR }}/combined_report.txt"
+
       - name: Send Slack Message
         if: ${{ inputs.send_slack_message && always() }}
         id: slack-message

scripts/combine_reports.py

@@ -0,0 +1,123 @@
+import json
+from io import StringIO
+from rich.table import Table
+from rich.console import Console
+import argparse
+import os
+
+
+def read_json_file(filename):
+
+    if not os.path.exists(filename):
+        return None
+
+    with open(filename, "r") as f:
+        data = json.load(f)
+    return data
+
+
+def extract_grype_data(data):
+    if data is None:
+        return []
+
+    matches = data["matches"]
+
+    out = []
+    for match in matches:
+        fixed = match["vulnerability"]["fix"]["versions"]
+        if len(fixed) > 0:
+            fixed = fixed[0]
+        else:
+            fixed = None
+        out.append({
+            "package": match["artifact"]["name"],
+            "version": match["artifact"]["version"],
+            "vulnerabilityID": match["vulnerability"]["id"],
+            "type": match["artifact"]["type"],
+            "fixed": fixed,
+            "severity": match["vulnerability"]["severity"].capitalize(),
+        })
+    return out
+
+
+def extract_trivy_data(data):
+    if data is None:
+        return []
+
+    out = []
+    results = data["Results"]
+    for result in results:
+        type = result["Type"]
+        matches = result["Vulnerabilities"]
+        for match in matches:
+            out.append({
+                "package": match["PkgName"],
+                "version": match["InstalledVersion"],
+                "vulnerabilityID": match["VulnerabilityID"],
+                "type": type.replace("python-pkg", "python"),
+                "fixed": None,
+                "severity": match["Severity"].capitalize(),
+            })
+    return out
+
+
+def combine_reports(grype_data, trivy_data):
+    out = grype_data.copy()
+
+    grype_keys = [(x["package"], x["version"], x["vulnerabilityID"]) for x in grype_data]
+    trivy_keys = [(x["package"], x["version"], x["vulnerabilityID"]) for x in trivy_data]
+
+    for key, item in zip(trivy_keys, trivy_data):
+        if key not in grype_keys:
+            out.append(item)
+
+    # sort by package name
+    out.sort(key=lambda x: x["package"])
+    return out
+
+
+def format_table(data):
+    table = Table(title="Vulnerabilities")
+    table.add_column("Package", justify="left")
+    table.add_column("Version", justify="left")
+    table.add_column("VulnerabilityID", justify="left")
+    table.add_column("Severity", justify="left")
+    table.add_column("Type", justify="left")
+    table.add_column("Fixed", justify="left")
+    for item in data:
+        table.add_row(
+            item["package"],
+            item["version"],
+            item["vulnerabilityID"],
+            item["severity"],
+            item["type"],
+            item["fixed"]
+        )
+    return table
+
+
+def save_table_to_file(table, filename):
+    console = Console(file=StringIO(), width=None, force_terminal=False)
+    console.print(table)
+    output = console.file.getvalue()
+    with open(filename, "w", encoding="utf-8") as f:
+        f.write(output)
+
+
+def main():
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("grype_file", type=str)
+    parser.add_argument("trivy_file", type=str)
+    args = parser.parse_args()
+
+    grype_data = extract_grype_data(read_json_file(args.grype_file))
+    trivy_data = extract_trivy_data(read_json_file(args.trivy_file))
+    combined_data = combine_reports(grype_data, trivy_data)
+    table = format_table(combined_data)
+    outdir = os.getenv("CVE_DIR", os.getcwd())
+    save_table_to_file(table, f"{outdir}/combined_report.txt")
+
+
+if __name__ == "__main__":
+    main()

scripts/install_runtime_requirements.sh

@@ -56,7 +56,7 @@ rm packages-microsoft-prod.deb
 apt-get update
 ACCEPT_EULA=Y apt-get install -y msodbcsql18 unixodbc-dev
 apt-get install -y libcurl4 libpython${PY_VERSION} openssl python3 python3-pip python3-setuptools \
-    adduser libgomp1 libaio1t64 libatomic1 libdw-dev --no-install-recommends
+    adduser libgomp1 libaio1t64 libatomic1 --no-install-recommends
 ln -s /usr/bin/$(ls /usr/bin | grep perf) /usr/bin/perf
 
 # In CI, we clean up the apt cache to save space in the Docker container.