Merge pull request github#6008 from github/tamasvajk/feature/csv-coverage-report

Add timeseries CSV generator script

Author: tamasvajk

.github/workflows/csv-coverage-timeseries.yml

@@ -0,0 +1,42 @@
+name: Build framework coverage timeseries reports
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Clone self (github/codeql)
+      uses: actions/checkout@v2
+      with:
+        path: script
+    - name: Clone self (github/codeql) for analysis
+      uses: actions/checkout@v2
+      with:
+        path: codeqlModels
+        fetch-depth: 0
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Download CodeQL CLI
+      env:
+         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+         gh release download --repo "github/codeql-cli-binaries" --pattern "codeql-linux64.zip"
+    - name: Unzip CodeQL CLI
+      run: unzip -d codeql-cli codeql-linux64.zip
+    - name: Build modeled package list
+      run: |
+        CLI=$(realpath "codeql-cli/codeql")
+        echo $CLI
+        PATH="$PATH:$CLI" python script/misc/scripts/library-coverage/generate-timeseries.py codeqlModels
+    - name: Upload timeseries CSV
+      uses: actions/upload-artifact@v2
+      with:
+        name: framework-coverage-timeseries
+        path: framework-coverage-timeseries-*.csv
+

.github/workflows/csv-coverage.yml

@@ -1,27 +1,11 @@
-name: Build/check CSV flow coverage report
+name: Build framework coverage reports
 
 on:
   workflow_dispatch:
     inputs:
       qlModelShaOverride:
         description: 'github/codeql repo SHA used for looking up the CSV models'
         required: false
-  push:
-    branches:
-     - main
-     - 'rc/**'
-  pull_request:
-    paths:
-      - '.github/workflows/csv-coverage.yml'
-      - '*/ql/src/**/*.ql'
-      - '*/ql/src/**/*.qll'
-      - 'misc/scripts/library-coverage/*.py'
-      # input data files
-      - '*/documentation/library-coverage/cwe-sink.csv'
-      - '*/documentation/library-coverage/frameworks.csv'
-      # coverage report files
-      - '*/documentation/library-coverage/flow-model-coverage.csv'
-      - '*/documentation/library-coverage/flow-model-coverage.rst'
 
 jobs:
   build:
@@ -33,28 +17,20 @@ jobs:
       uses: actions/checkout@v2
       with:
         path: script
-    - name: Clone self (github/codeql) at a given SHA for analysis
-      if: github.event.inputs.qlModelShaOverride != ''
-      uses: actions/checkout@v2
-      with:
-        path: codeqlModels
-        ref: github.event.inputs.qlModelShaOverride
     - name: Clone self (github/codeql) for analysis
-      if: github.event.inputs.qlModelShaOverride == ''
       uses: actions/checkout@v2
       with:
         path: codeqlModels
+        ref: ${{ github.event.inputs.qlModelShaOverride || github.ref }}
     - name: Set up Python 3.8
       uses: actions/setup-python@v2
       with:
         python-version: 3.8
     - name: Download CodeQL CLI
-      uses: dsaltares/fetch-gh-release-asset@aa37ae5c44d3c9820bc12fe675e8670ecd93bd1c
-      with:
-        repo: "github/codeql-cli-binaries"
-        version: "latest"
-        file: "codeql-linux64.zip"
-        token: ${{ secrets.GITHUB_TOKEN }}
+      env:
+         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+         gh release download --repo "github/codeql-cli-binaries" --pattern "codeql-linux64.zip"
     - name: Unzip CodeQL CLI
       run: unzip -d codeql-cli codeql-linux64.zip
     - name: Build modeled package list
@@ -63,15 +39,11 @@ jobs:
     - name: Upload CSV package list
       uses: actions/upload-artifact@v2
       with:
-        name: csv-flow-model-coverage
-        path: flow-model-coverage-*.csv
+        name: framework-coverage-csv
+        path: framework-coverage-*.csv
     - name: Upload RST package list
       uses: actions/upload-artifact@v2
       with:
-        name: rst-flow-model-coverage
-        path: flow-model-coverage-*.rst
-    # - name: Check coverage files
-    #   if: github.event.pull_request
-    #   run: |
-    #     python script/misc/scripts/library-coverage/compare-files.py codeqlModels
+        name: framework-coverage-rst
+        path: framework-coverage-*.rst
 

java/documentation/library-coverage/coverage.csv

@@ -0,0 +1,44 @@
+package,sink,source,summary,sink:bean-validation,sink:create-file,sink:header-splitting,sink:information-leak,sink:jexl,sink:ldap,sink:open-url,sink:set-hostname-verifier,sink:url-open-stream,sink:xpath,sink:xss,source:remote,summary:taint,summary:value
+android.util,,16,,,,,,,,,,,,,16,,
+android.webkit,3,2,,,,,,,,,,,,3,2,,
+com.esotericsoftware.kryo.io,,,1,,,,,,,,,,,,,1,
+com.esotericsoftware.kryo5.io,,,1,,,,,,,,,,,,,1,
+com.fasterxml.jackson.databind,,,3,,,,,,,,,,,,,3,
+com.google.common.base,,,34,,,,,,,,,,,,,28,6
+com.google.common.io,6,,73,,,,,,,,,6,,,,72,1
+com.unboundid.ldap.sdk,17,,,,,,,,17,,,,,,,,
+java.beans,,,1,,,,,,,,,,,,,1,
+java.io,3,,20,,3,,,,,,,,,,,20,
+java.lang,,,3,,,,,,,,,,,,,1,2
+java.net,2,3,4,,,,,,,2,,,,,3,4,
+java.nio,10,,2,,10,,,,,,,,,,,2,
+java.util,,,283,,,,,,,,,,,,,15,268
+javax.naming.directory,1,,,,,,,,1,,,,,,,,
+javax.net.ssl,2,,,,,,,,,,2,,,,,,
+javax.servlet,4,21,2,,,3,1,,,,,,,,21,2,
+javax.validation,1,1,,1,,,,,,,,,,,1,,
+javax.ws.rs.core,1,,,,,1,,,,,,,,,,,
+javax.xml.transform.sax,,,4,,,,,,,,,,,,,4,
+javax.xml.transform.stream,,,2,,,,,,,,,,,,,2,
+javax.xml.xpath,3,,,,,,,,,,,,3,,,,
+org.apache.commons.codec,,,2,,,,,,,,,,,,,2,
+org.apache.commons.io,,,22,,,,,,,,,,,,,22,
+org.apache.commons.jexl2,15,,,,,,,15,,,,,,,,,
+org.apache.commons.jexl3,15,,,,,,,15,,,,,,,,,
+org.apache.commons.lang3,,,370,,,,,,,,,,,,,324,46
+org.apache.commons.text,,,272,,,,,,,,,,,,,220,52
+org.apache.directory.ldap.client.api,1,,,,,,,,1,,,,,,,,
+org.apache.hc.core5.function,,,1,,,,,,,,,,,,,1,
+org.apache.hc.core5.http,1,2,39,,,,,,,,,,,1,2,39,
+org.apache.hc.core5.net,,,2,,,,,,,,,,,,,2,
+org.apache.hc.core5.util,,,24,,,,,,,,,,,,,18,6
+org.apache.http,2,3,67,,,,,,,,,,,2,3,59,8
+org.dom4j,20,,,,,,,,,,,,20,,,,
+org.springframework.ldap.core,14,,,,,,,,14,,,,,,,,
+org.springframework.security.web.savedrequest,,6,,,,,,,,,,,,,6,,
+org.springframework.web.client,,3,,,,,,,,,,,,,3,,
+org.springframework.web.context.request,,8,,,,,,,,,,,,,8,,
+org.springframework.web.multipart,,12,,,,,,,,,,,,,12,,
+org.xml.sax,,,1,,,,,,,,,,,,,1,
+org.xmlpull.v1,,3,,,,,,,,,,,,,3,,
+play.mvc,,4,,,,,,,,,,,,,4,,

java/documentation/library-coverage/flow-model-coverage.rst renamed to java/documentation/library-coverage/coverage.rst

@@ -8,12 +8,14 @@ Java framework & library support
 
    Framework / library,Package,Remote flow sources,Taint & value steps,Sinks (total),`CWE‑022` :sub:`Path injection`,`CWE‑036` :sub:`Path traversal`,`CWE‑079` :sub:`Cross-site scripting`,`CWE‑089` :sub:`SQL injection`,`CWE‑090` :sub:`LDAP injection`,`CWE‑094` :sub:`Code injection`,`CWE‑319` :sub:`Cleartext transmission`
    Android,``android.*``,18,,3,,,3,,,,
-   Apache,``org.apache.*``,5,648,4,,,3,,1,,
    `Apache Commons IO <https://commons.apache.org/proper/commons-io/>`_,``org.apache.commons.io``,,22,,,,,,,,
-   Google,``com.google.common.*``,,97,6,,6,,,,,
-   Java Standard Library,``java.*``,3,41,15,13,,,,,,2
+   `Apache Commons Lang <https://commons.apache.org/proper/commons-lang/>`_,``org.apache.commons.lang3``,,370,,,,,,,,
+   `Apache Commons Text <https://commons.apache.org/proper/commons-text/>`_,``org.apache.commons.text``,,272,,,,,,,,
+   `Apache HttpComponents <https://hc.apache.org/>`_,"``org.apache.hc.core5.*``, ``org.apache.http``",5,133,3,,,3,,,,
+   `Google Guava <https://guava.dev/>`_,``com.google.common.*``,,107,6,,6,,,,,
+   Java Standard Library,``java.*``,3,313,15,13,,,,,,2
    Java extensions,``javax.*``,22,8,12,,,,,1,1,
    `Spring <https://spring.io/>`_,``org.springframework.*``,29,,14,,,,,14,,
-   Others,"``com.esotericsoftware.kryo.io``, ``com.esotericsoftware.kryo5.io``, ``com.fasterxml.jackson.databind``, ``com.unboundid.ldap.sdk``, ``org.dom4j``, ``org.xml.sax``, ``org.xmlpull.v1``, ``play.mvc``",7,5,37,,,,,17,,
-   Totals,,84,821,91,13,6,6,,33,1,2
+   Others,"``com.esotericsoftware.kryo.io``, ``com.esotericsoftware.kryo5.io``, ``com.fasterxml.jackson.databind``, ``com.unboundid.ldap.sdk``, ``org.apache.commons.codec``, ``org.apache.commons.jexl2``, ``org.apache.commons.jexl3``, ``org.apache.directory.ldap.client.api``, ``org.dom4j``, ``org.xml.sax``, ``org.xmlpull.v1``, ``play.mvc``",7,8,68,,,,,18,,
+   Totals,,84,1233,121,13,6,6,,33,1,2
 

java/documentation/library-coverage/flow-model-coverage.csv

@@ -1,8 +1,10 @@
-Framework name,URL,Package prefix
+Framework name,URL,Package prefixes
 Java Standard Library,,java.*
-Google,,com.google.common.*
-Apache,,org.apache.*
+Java extensions,,javax.*
+Google Guava,https://guava.dev/,com.google.common.*
 Apache Commons IO,https://commons.apache.org/proper/commons-io/,org.apache.commons.io
+Apache Commons Lang,https://commons.apache.org/proper/commons-lang/,org.apache.commons.lang3
+Apache Commons Text,https://commons.apache.org/proper/commons-text/,org.apache.commons.text
+Apache HttpComponents,https://hc.apache.org/,org.apache.hc.core5.* org.apache.http
 Android,,android.*
-Spring,https://spring.io/,org.springframework.*
-Java extensions,,javax.*
+Spring,https://spring.io/,org.springframework.*

java/documentation/library-coverage/frameworks.csv

@@ -0,0 +1,75 @@
+import csv
+import sys
+import packages
+
+
+class Framework:
+    """
+    Frameworks are the aggregation units in the RST and timeseries report. These are read from the frameworks.csv file.
+    """
+
+    def __init__(self, name, url, package_pattern):
+        self.name = name
+        self.url = url
+        self.package_pattern = package_pattern
+
+
+class FrameworkCollection:
+    """
+    A (sorted) list of frameworks.
+    """
+
+    def __init__(self, path):
+        self.frameworks: list[Framework] = []
+        self.package_patterns = set()
+
+        with open(path) as csvfile:
+            reader = csv.reader(csvfile)
+            next(reader)
+            for row in reader:
+                # row: Hibernate,https://hibernate.org/,org.hibernate
+                self.__add(Framework(row[0], row[1], row[2]))
+        self.__sort()
+
+    def __add(self, framework: Framework):
+        if framework.package_pattern not in self.package_patterns:
+            self.package_patterns.add(framework.package_pattern)
+            self.frameworks.append(framework)
+        else:
+            print("Package pattern already exists: " +
+                  framework.package_pattern, file=sys.stderr)
+
+    def __sort(self):
+        self.frameworks.sort(key=lambda f: f.name)
+
+    def get(self, framework_name):
+        for framework in self.frameworks:
+            if framework.name == framework_name:
+                return framework
+        return None
+
+    def get_frameworks(self):
+        return self.frameworks
+
+    def __package_match_single(self, package: packages.Package, pattern):
+        return (pattern.endswith("*") and package.name.startswith(pattern[:-1])) or (not pattern.endswith("*") and pattern == package.name)
+
+    def __package_match(self, package: packages.Package, pattern):
+        patterns = pattern.split(" ")
+        return any(self.__package_match_single(package, pattern) for pattern in patterns)
+
+    def get_package_filter(self, framework: Framework):
+        """
+        Returns a lambda filter that holds for packages that match the current framework.
+
+        The pattern is either full name, such as "org.hibernate", or a prefix, such as "java.*".
+        Patterns can also contain a space separated list of patterns, such as "java.sql javax.swing".
+
+        Package patterns might overlap, in case of 'org.apache.commons.io' and 'org.apache.*', the statistics for
+        the latter will not include the statistics for the former.
+        """
+        return lambda p: \
+            self.__package_match(p, framework.package_pattern) and \
+            all(
+                len(framework.package_pattern) >= len(pattern) or
+                not self.__package_match(p, pattern) for pattern in self.package_patterns)