Automation to regenerate framework models

Author: Benjamin Muskalla

.github/workflows/mad_regenerate-models.yml

@@ -0,0 +1,56 @@
+name: Regenerate framework models
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "30 2 * * *"
+
+jobs:
+  regenerate-models:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        slug: ["placeholder"]
+        ref: ["placeholder"]
+        include:
+          - slug: "apache/commons-io"
+            ref: "8985de8fe74f6622a419b37a6eed0dbc484dc128"
+        exclude:
+          - slug: "placeholder"
+            ref: "placeholder"
+    steps:
+      - name: Clone self (github/codeql)
+        uses: actions/checkout@v2
+      - name: Setup CodeQL binaries
+        uses: ./.github/actions/fetch-codeql
+      - name: Clone repositories
+        uses: actions/checkout@v2
+        with:
+          path: repos/${{ matrix.ref }}
+          ref: ${{ matrix.ref }}
+          repository: ${{ matrix.slug }}
+      - name: Build database
+        env:
+          SLUG: ${{ matrix.slug }}
+          REF: ${{ matrix.ref }}
+        run: |
+          mkdir dbs
+          cd repos/${REF}
+          SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
+          codeql database create --language=java ../../dbs/${SHORTNAME}
+      - name: Regenerate models in-place
+        env:
+          SLUG: ${{ matrix.slug }}
+        run: |
+          SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
+          java/ql/src/utils/model-generator/RegenerateModels.py "${SLUG}" dbs/${SHORTNAME}
+      - name: Stage changes
+        run: |
+          find java -name "*.qll" -print0 | xargs -0 git add
+          git status
+          git diff --cached > models.patch
+      - uses: actions/upload-artifact@v2
+        with:
+          name: patch
+          path: models.patch
+          retention-days: 7

java/ql/src/utils/model-generator/RegenerateModels.py

@@ -0,0 +1,47 @@
+#!/usr/bin/python3
+
+# Tool to regenerate existing framework CSV models.
+
+from pathlib import Path
+import json
+import os
+import requests
+import shutil
+import subprocess
+import tempfile
+import sys
+
+
+lgtmSlugToModelFile = {
+    # "apache/commons-beanutils": "java/ql/lib/semmle/code/java/frameworks/apache/BeanUtilsGenerated.qll",
+    # "apache/commons-codec": "java/ql/lib/semmle/code/java/frameworks/apache/CodecGenerated.qll",
+    # "apache/commons-lang": "java/ql/lib/semmle/code/java/frameworks/apache/Lang3Generated.qll",
+    "apache/commons-io": "java/ql/lib/semmle/code/java/frameworks/apache/IOGenerated.qll",
+}
+
+
+def findGitRoot():
+    return subprocess.check_output(
+        ["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip()
+
+
+def regenerateModel(lgtmSlug, extractedDb):
+    tmpDir = tempfile.mkdtemp()
+    print("============================================================")
+    print("Generating models for " + lgtmSlug)
+    print("============================================================")
+    modelFile = lgtmSlugToModelFile[lgtmSlug]
+    codeQlRoot = findGitRoot()
+    targetModel = codeQlRoot + "/" + modelFile
+    subprocess.check_call([codeQlRoot + "/java/ql/src/utils/model-generator/GenerateFlowModel.py", extractedDb,
+                           targetModel])
+    print("Regenerated " + targetModel)
+    shutil.rmtree(tmpDir)
+
+
+if len(sys.argv) == 3:
+    lgtmSlug = sys.argv[1]
+    db = sys.argv[2]
+    regenerateModel(lgtmSlug, db)
+else:
+    print('error')