Merge pull request github#7767 from bmuskalla/regenerateModelScript

Java: Regenerate framework models automatically

Author: Benjamin Muskalla

.github/workflows/mad_regenerate-models.yml

@@ -0,0 +1,62 @@
+name: Regenerate framework models
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "30 2 * * *"
+  pull_request:
+    branches:
+      - main
+    paths:
+      - ".github/workflows/mad_regenerate-models.yml"
+
+jobs:
+  regenerate-models:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # placeholder required for each axis, excluded below, replaced by the actual combinations (see include)
+        slug: ["placeholder"]
+        ref: ["placeholder"]
+        include:
+          - slug: "apache/commons-io"
+            ref: "8985de8fe74f6622a419b37a6eed0dbc484dc128"
+        exclude:
+          - slug: "placeholder"
+            ref: "placeholder"
+    steps:
+      - name: Clone self (github/codeql)
+        uses: actions/checkout@v2
+      - name: Setup CodeQL binaries
+        uses: ./.github/actions/fetch-codeql
+      - name: Clone repositories
+        uses: actions/checkout@v2
+        with:
+          path: repos/${{ matrix.ref }}
+          ref: ${{ matrix.ref }}
+          repository: ${{ matrix.slug }}
+      - name: Build database
+        env:
+          SLUG: ${{ matrix.slug }}
+          REF: ${{ matrix.ref }}
+        run: |
+          mkdir dbs
+          cd repos/${REF}
+          SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
+          codeql database create --language=java ../../dbs/${SHORTNAME}
+      - name: Regenerate models in-place
+        env:
+          SLUG: ${{ matrix.slug }}
+        run: |
+          SHORTNAME=${SLUG//[^a-zA-Z0-9_]/}
+          java/ql/src/utils/model-generator/RegenerateModels.py "${SLUG}" dbs/${SHORTNAME}
+      - name: Stage changes
+        run: |
+          find java -name "*.qll" -print0 | xargs -0 git add
+          git status
+          git diff --cached > models.patch
+      - uses: actions/upload-artifact@v2
+        with:
+          name: patch
+          path: models.patch
+          retention-days: 7

java/ql/src/utils/model-generator/RegenerateModels.py

@@ -0,0 +1,53 @@
+#!/usr/bin/python3
+
+# Tool to regenerate existing framework CSV models.
+
+from pathlib import Path
+import json
+import os
+import requests
+import shutil
+import subprocess
+import tempfile
+import sys
+
+
+defaultModelPath = "java/ql/lib/semmle/code/java/frameworks"
+lgtmSlugToModelFile = {
+    # "apache/commons-beanutils": "apache/BeanUtilsGenerated.qll",
+    # "apache/commons-codec": "apache/CodecGenerated.qll",
+    # "apache/commons-lang": "apache/Lang3Generated.qll",
+    "apache/commons-io": "apache/IOGenerated.qll",
+}
+
+
+def findGitRoot():
+    return subprocess.check_output(
+        ["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip()
+
+
+def regenerateModel(lgtmSlug, extractedDb):
+    tmpDir = tempfile.mkdtemp()
+    print("============================================================")
+    print("Generating models for " + lgtmSlug)
+    print("============================================================")
+    # check if lgtmSlug exists as key
+    if lgtmSlug not in lgtmSlugToModelFile:
+        print("ERROR: slug " + lgtmSlug +
+              " is not mapped to a model file in script " + sys.argv[0])
+        sys.exit(1)
+    modelFile = defaultModelPath + "/" + lgtmSlugToModelFile[lgtmSlug]
+    codeQlRoot = findGitRoot()
+    targetModel = codeQlRoot + "/" + modelFile
+    subprocess.check_call([codeQlRoot + "/java/ql/src/utils/model-generator/GenerateFlowModel.py", extractedDb,
+                           targetModel])
+    print("Regenerated " + targetModel)
+    shutil.rmtree(tmpDir)
+
+
+if len(sys.argv) == 3:
+    lgtmSlug = sys.argv[1]
+    db = sys.argv[2]
+    regenerateModel(lgtmSlug, db)
+else:
+    print('error')