Merge pull request github#5755 from RasmusWL/non-alert-data-part1

Approved by tausbn

Author: codeql-ci

python/ql/src/Diagnostics/ExtractionErrors.ql

@@ -0,0 +1,23 @@
+/**
+ * @name Python extraction errors
+ * @description List all extraction errors for Python files in the source code directory.
+ * @kind diagnostic
+ * @id py/diagnostics/extraction-errors
+ */
+
+import python
+
+/**
+ * Gets the SARIF severity for errors.
+ *
+ * See point 3.27.10 in https://docs.oasis-open.org/sarif/sarif/v2.0/sarif-v2.0.html for
+ * what error means.
+ */
+int getErrorSeverity() { result = 2 }
+
+from SyntaxError error, File file
+where
+  file = error.getFile() and
+  exists(file.getRelativePath())
+select error, "Extraction failed in " + file + " with error " + error.getMessage(),
+  getErrorSeverity()

python/ql/src/Diagnostics/SuccessfullyExtractedFiles.ql

@@ -0,0 +1,15 @@
+/**
+ * @name Successfully extracted Python files
+ * @description Lists all Python files in the source code directory that were extracted
+ *   without encountering an error.
+ * @kind diagnostic
+ * @id py/diagnostics/successfully-extracted-files
+ */
+
+import python
+
+from File file
+where
+  not exists(SyntaxError e | e.getFile() = file) and
+  exists(file.getRelativePath())
+select file, ""

python/ql/src/Summary/LinesOfCode.ql

@@ -0,0 +1,13 @@
+/**
+ * @name Total lines of Python code in the database
+ * @description The total number of lines of Python code across all files, including
+ *   external libraries and auto-generated files. This is a useful metric of the size of a
+ *   database. This query counts the lines of code, excluding whitespace or comments.
+ * @kind metric
+ * @tags summary
+ * @id py/summary/lines-of-code
+ */
+
+import python
+
+select sum(Module m | | m.getMetrics().getNumberOfLinesOfCode())

python/ql/src/Summary/LinesOfUserCode.ql

@@ -0,0 +1,21 @@
+/**
+ * @name Total lines of user written Python code in the database
+ * @description The total number of lines of Python code from the source code directory,
+ *   excluding auto-generated files. This query counts the lines of code, excluding
+ *   whitespace or comments. Note: If external libraries are included in the codebase
+ *   either in a checked-in virtual environment or as vendored code, that will currently
+ *   be counted as user written code.
+ * @kind metric
+ * @tags summary
+ * @id py/summary/lines-of-user-code
+ */
+
+import python
+import semmle.python.filters.GeneratedCode
+
+select sum(Module m |
+    exists(m.getFile().getRelativePath()) and
+    not m.getFile() instanceof GeneratedFile
+  |
+    m.getMetrics().getNumberOfLinesOfCode()
+  )

python/ql/test/query-tests/Diagnostics/ExtractionErrors.expected

@@ -0,0 +1,2 @@
+| bad_encoding.py:2:11:2:11 | Encoding Error | Extraction failed in bad_encoding.py with error 'utf-8' codec can't decode byte 0x9d in position 87: invalid start byte | 2 |
+| syntax_error.py:1:31:1:31 | Syntax Error | Extraction failed in syntax_error.py with error Syntax Error | 2 |

python/ql/test/query-tests/Diagnostics/ExtractionErrors.qlref

@@ -0,0 +1 @@
+Diagnostics/ExtractionErrors.ql

python/ql/test/query-tests/Diagnostics/SuccessfullyExtractedFiles.expected

@@ -0,0 +1 @@
+| good_file.py:0:0:0:0 | good_file.py |  |

python/ql/test/query-tests/Diagnostics/SuccessfullyExtractedFiles.qlref

@@ -0,0 +1 @@
+Diagnostics/SuccessfullyExtractedFiles.ql

python/ql/test/query-tests/Diagnostics/bad_encoding.py

@@ -0,0 +1,2 @@
+# Note: This file has been encoded in Windows 1252 to provoke encoding error
+print("wat")

python/ql/test/query-tests/Diagnostics/good_file.py

@@ -0,0 +1 @@
+print("hello world")