Shared: Remove the language-specific model generator scripts

Author: paldepind

.github/workflows/mad_modelDiff.yml

@@ -68,7 +68,7 @@ jobs:
             DATABASE=$2
             cd codeql-$QL_VARIANT
             SHORTNAME=`basename $DATABASE`
-            python java/ql/src/utils/modelgenerator/GenerateFlowModel.py --with-summaries --with-sinks $DATABASE $SHORTNAME/$QL_VARIANT
+            python misc/scripts/models-as-data/generate_mad.py --language java --with-summaries --with-sinks $DATABASE $SHORTNAME/$QL_VARIANT
             mkdir -p $MODELS/$SHORTNAME
             mv java/ql/lib/ext/generated/$SHORTNAME/$QL_VARIANT $MODELS/$SHORTNAME
             cd ..

cpp/ql/src/utils/modelgenerator/GenerateFlowModel.py

@@ -0,0 +1,8 @@
+---
+category: minorAnalysis
+---
+* Changes to the MaD model generation infrastructure:
+  * The `csharp/ql/src/utils/modelgenerator/GenerateFlowModel.py` script has
+    been removed. The `/misc/scripts/models-as-data/generate_mad.py` script now
+    supports being called directly and should be used instead. The script
+    requires a `--language` argument but otherwise functions identically.

csharp/ql/src/change-notes/2025-05-02-mad-generator-renamed.md

@@ -0,0 +1,8 @@
+---
+category: minorAnalysis
+---
+* Changes to the MaD model generation infrastructure:
+  * The `java/ql/src/utils/modelgenerator/GenerateFlowModel.py` script has
+    been removed. The `/misc/scripts/models-as-data/generate_mad.py` script now
+    supports being called directly and should be used instead. The script
+    requires a `--language` argument but otherwise functions identically.

csharp/ql/src/utils/modelgenerator/GenerateFlowModel.py

@@ -35,8 +35,8 @@ def regenerateModel(lgtmSlug, extractedDb):
         sys.exit(1)
     modelFile = lgtmSlugToModelFile[lgtmSlug]
     codeQlRoot = findGitRoot()
-    subprocess.check_call([codeQlRoot + "/java/ql/src/utils/modelgenerator/GenerateFlowModel.py",
-                           "--with-summaries", "--with-sinks", "--with-neutrals",
+    subprocess.check_call([codeQlRoot + "/misc/scripts/models-as-data/generate_mad.py",
+                           "--language", "java", "--with-summaries", "--with-sinks", "--with-neutrals",
                            extractedDb, modelFile])
     print("Regenerated " + modelFile)
     shutil.rmtree(tmpDir)

java/ql/src/change-notes/2025-05-02-mad-generator-renamed.md

@@ -1,10 +1,8 @@
 #!/usr/bin/python3
 
 import helpers
-import json
 import os
 import os.path
-import shlex
 import subprocess
 import sys
 import tempfile
@@ -27,24 +25,13 @@ def parseData(data):
 
     return rows
 
-class Generator:
-    def __init__ (self, language):
-        self.language = language
-        self.generateSinks = False
-        self.generateSources = False
-        self.generateSummaries = False
-        self.generateNeutrals = False
-        self.generateTypeBasedSummaries = False
-        self.dryRun = False
-        self.dirname = "modelgenerator"
-
 
-    def printHelp(self):
-        print(f"""Usage:
-python3 GenerateFlowModel.py <library-database> [DIR] [--with-sinks] [--with-sources] [--with-summaries] [--with-neutrals] [--with-typebased-summaries] [--dry-run]
+def printHelp():
+    print(f"""Usage:
+python3 generate_mad.py <library-database> [DIR] --language LANGUAGE [--with-sinks] [--with-sources] [--with-summaries] [--with-neutrals] [--with-typebased-summaries] [--dry-run]
 
 This generates summary, source, sink and neutral models for the code in the database.
-The files will be placed in `{self.language}/ql/lib/ext/generated/DIR`
+The files will be placed in `LANGUAGE/ql/lib/ext/generated/DIR`
 
 Which models are generated is controlled by the flags:
     --with-sinks
@@ -57,14 +44,25 @@ def printHelp(self):
     --dry-run: Only run the queries, but don't write to file.
 
 Example invocations:
-$ python3 GenerateFlowModel.py /tmp/dbs/my_library_db
-$ python3 GenerateFlowModel.py /tmp/dbs/my_library_db --with-sinks
-$ python3 GenerateFlowModel.py /tmp/dbs/my_library_db --with-sinks my_directory
+$ python3 generate_mad.py /tmp/dbs/my_library_db
+$ python3 generate_mad.py /tmp/dbs/my_library_db --with-sinks
+$ python3 generate_mad.py /tmp/dbs/my_library_db --with-sinks my_directory
 
 
 Requirements: `codeql` should appear on your path.
     """)
 
+class Generator:
+    def __init__(self, language):
+        self.language = language
+        self.generateSinks = False
+        self.generateSources = False
+        self.generateSummaries = False
+        self.generateNeutrals = False
+        self.generateTypeBasedSummaries = False
+        self.dryRun = False
+        self.dirname = "modelgenerator"
+
 
     def setenvironment(self, database, folder):
         self.codeQlRoot = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip()
@@ -76,12 +74,22 @@ def setenvironment(self, database, folder):
 
 
     @staticmethod
-    def make(language):
-        generator = Generator(language)
+    def make():
+        '''Create a generator instance based on command line arguments.'''
         if any(s == "--help" for s in sys.argv):
-            generator.printHelp()
+            printHelp()
+            sys.exit(0)
+
+        if "--language" in sys.argv:
+            language = sys.argv[sys.argv.index("--language") + 1]
+            sys.argv.remove("--language")
+            sys.argv.remove(language)
+        else:
+            print("Error: Language not specified. Use --language <language>.")
             sys.exit(0)
 
+        generator = Generator(language=language)
+
         if "--with-sinks" in sys.argv:
             sys.argv.remove("--with-sinks")
             generator.generateSinks = True
@@ -115,7 +123,7 @@ def make(language):
 
         n = len(sys.argv)
         if n < 2:
-            generator.printHelp()
+            printHelp()
             sys.exit(1)
         elif n == 2:
             generator.setenvironment(sys.argv[1], "")
@@ -204,3 +212,6 @@ def run(self):
 
         if self.generateTypeBasedSummaries:
             self.save(typeBasedContent, ".typebased.model.yml")
+
+if __name__ == '__main__':
+    Generator.make().run()