Merge branch 'main' into topPack

Author: erik-krogh

.github/workflows/check-change-note.yml

@@ -1,3 +1,5 @@
+name: Check change note
+
 on:
   pull_request_target:
     types: [labeled, unlabeled, opened, synchronize, reopened, ready_for_review]

.github/workflows/docs-review.yml

@@ -17,7 +17,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Build" Version="16.0.461" />
+    <PackageReference Include="Microsoft.Build" Version="16.9.0" />
   </ItemGroup>
 
   <ItemGroup>

cpp/autobuilder/Semmle.Autobuild.Cpp/Semmle.Autobuild.Cpp.csproj

@@ -0,0 +1,2 @@
+lgtm,codescanning
+* The 'Resource not released in destructor' (cpp/resource-not-released-in-destructor) query has been improved to recognize more releases of resources.

cpp/change-notes/2021-03-17-av-rule-79.md

@@ -0,0 +1,16 @@
+/**
+ * @name Extraction errors
+ * @description List all extraction errors for files in the source code directory.
+ * @kind diagnostic
+ * @id cpp/diagnostics/extraction-errors
+ */
+
+import cpp
+import ExtractionErrors
+
+from ExtractionError error
+where
+  error instanceof ExtractionUnknownError or
+  exists(error.getFile().getRelativePath())
+select error, "Extraction failed in " + error.getFile() + " with error " + error.getErrorMessage(),
+  error.getSeverity()

cpp/ql/src/Diagnostics/ExtractionErrors.ql

@@ -0,0 +1,137 @@
+/**
+ * Provides a common hierarchy of all types of errors that can occur during extraction.
+ */
+
+import cpp
+
+/*
+ * A note about how the C/C++ extractor emits diagnostics:
+ * When the extractor frontend encounters an error, it emits a diagnostic message,
+ * that includes a message, location and severity.
+ * However, that process is best-effort and may fail (e.g. due to lack of memory).
+ * Thus, if the extractor emitted at least one diagnostic of severity discretionary
+ * error (or higher), it *also* emits a simple "There was an error during this compilation"
+ * error diagnostic, without location information.
+ * In the common case, this means that a compilation during which one or more errors happened also gets
+ * the catch-all diagnostic.
+ * This diagnostic has the empty string as file path.
+ * We filter out these useless diagnostics if there is at least one error-level diagnostic
+ * for the affected compilation in the database.
+ * Otherwise, we show it to indicate that something went wrong and that we
+ * don't know what exactly happened.
+ */
+
+/**
+ * An error that, if present, leads to a file being marked as non-successfully extracted.
+ */
+class ReportableError extends Diagnostic {
+  ReportableError() {
+    (
+      this instanceof CompilerDiscretionaryError or
+      this instanceof CompilerError or
+      this instanceof CompilerCatastrophe
+    ) and
+    // Filter for the catch-all diagnostic, see note above.
+    not this.getFile().getAbsolutePath() = ""
+  }
+}
+
+private newtype TExtractionError =
+  TReportableError(ReportableError err) or
+  TCompilationFailed(Compilation c, File f) {
+    f = c.getAFileCompiled() and not c.normalTermination()
+  } or
+  // Show the catch-all diagnostic (see note above) only if we haven't seen any other error-level diagnostic
+  // for that compilation
+  TUnknownError(CompilerError err) {
+    not exists(ReportableError e | e.getCompilation() = err.getCompilation())
+  }
+
+/**
+ * Superclass for the extraction error hierarchy.
+ */
+class ExtractionError extends TExtractionError {
+  /** Gets the string representation of the error. */
+  string toString() { none() }
+
+  /** Gets the error message for this error. */
+  string getErrorMessage() { none() }
+
+  /** Gets the file this error occured in. */
+  File getFile() { none() }
+
+  /** Gets the location this error occured in. */
+  Location getLocation() { none() }
+
+  /** Gets the SARIF severity of this error. */
+  int getSeverity() {
+    // Unfortunately, we can't distinguish between errors and fatal errors in SARIF,
+    // so all errors have severity 2.
+    result = 2
+  }
+}
+
+/**
+ * An unrecoverable extraction error, where extraction was unable to finish.
+ * This can be caused by a multitude of reasons, for example:
+ *  - hitting a frontend assertion
+ *  - crashing due to dereferencing an invalid pointer
+ *  - stack overflow
+ *  - out of memory
+ */
+class ExtractionUnrecoverableError extends ExtractionError, TCompilationFailed {
+  Compilation c;
+  File f;
+
+  ExtractionUnrecoverableError() { this = TCompilationFailed(c, f) }
+
+  override string toString() {
+    result = "Unrecoverable extraction error while compiling " + f.toString()
+  }
+
+  override string getErrorMessage() { result = "unrecoverable compilation failure." }
+
+  override File getFile() { result = f }
+
+  override Location getLocation() { result = f.getLocation() }
+}
+
+/**
+ * A recoverable extraction error.
+ * These are compiler errors from the frontend.
+ * Upon encountering one of these, we still continue extraction, but the
+ * database will be incomplete for that file.
+ */
+class ExtractionRecoverableError extends ExtractionError, TReportableError {
+  ReportableError err;
+
+  ExtractionRecoverableError() { this = TReportableError(err) }
+
+  override string toString() { result = "Recoverable extraction error: " + err }
+
+  override string getErrorMessage() { result = err.getFullMessage() }
+
+  override File getFile() { result = err.getFile() }
+
+  override Location getLocation() { result = err.getLocation() }
+}
+
+/**
+ * An unknown error happened during extraction.
+ * These are only displayed if we know that we encountered an error during extraction,
+ * but, for some reason, failed to emit a proper diagnostic with location information
+ * and error message.
+ */
+class ExtractionUnknownError extends ExtractionError, TUnknownError {
+  CompilerError err;
+
+  ExtractionUnknownError() { this = TUnknownError(err) }
+
+  override string toString() { result = "Unknown extraction error: " + err }
+
+  override string getErrorMessage() { result = err.getFullMessage() }
+
+  override File getFile() { result = err.getFile() }
+
+  override Location getLocation() { result = err.getLocation() }
+}

cpp/ql/src/Diagnostics/ExtractionErrors.qll

@@ -1,8 +1,8 @@
 /**
- * @name Failed extractions
- * @description Gives the command-line of compilations for which extraction did not run to completion.
+ * @name Failed extractor invocations
+ * @description Gives the command line of compilations for which extraction did not run to completion.
  * @kind diagnostic
- * @id cpp/diagnostics/failed-extractions
+ * @id cpp/diagnostics/failed-extractor-invocations
  */
 
 import cpp
@@ -19,4 +19,4 @@ string describe(Compilation c) {
 
 from Compilation c
 where not c.normalTermination()
-select c, "Extraction failed for " + describe(c), 2
+select c, "Extraction aborted for " + describe(c), 2

cpp/ql/src/Diagnostics/FailedExtractions.ql renamed to cpp/ql/src/Diagnostics/FailedExtractorInvocations.ql

@@ -0,0 +1,15 @@
+/**
+ * @name Successfully extracted files
+ * @description Lists all files in the source code directory that were extracted without encountering an error in the file.
+ * @kind diagnostic
+ * @id cpp/diagnostics/successfully-extracted-files
+ */
+
+import cpp
+import ExtractionErrors
+
+from File f
+where
+  not exists(ExtractionError e | e.getFile() = f) and
+  exists(f.getRelativePath())
+select f, ""

cpp/ql/src/Diagnostics/SuccessfullyExtractedFiles.ql

@@ -0,0 +1,11 @@
+/**
+ * @id cpp/summary/lines-of-code
+ * @name Total lines of C/C++ code in the database
+ * @description The total number of lines of C/C++ code across all files, including system headers, libraries, and auto-generated files. This is a useful metric of the size of a database. For all files that were seen during the build, this query counts the lines of code, excluding whitespace or comments.
+ * @kind metric
+ * @tags summary
+ */
+
+import cpp
+
+select sum(File f | f.fromSource() | f.getMetrics().getNumberOfLinesOfCode())

cpp/ql/src/Summary/LinesOfCode.ql

@@ -13,6 +13,7 @@
 
 import cpp
 import Critical.NewDelete
+import semmle.code.cpp.valuenumbering.GlobalValueNumbering
 
 /**
  * An expression that acquires a resource, and the kind of resource that is acquired.  The
@@ -98,7 +99,8 @@ private predicate exprReleases(Expr e, Expr released, string kind) {
       e.(FunctionCall).getTarget().(MemberFunction).getAnOverridingFunction+() = f
     ) and
     e.(FunctionCall).getArgument(arg) = released and
-    exprReleases(_, exprOrDereference(f.getParameter(arg).getAnAccess()), kind)
+    exprReleases(_,
+      exprOrDereference(globalValueNumber(f.getParameter(arg).getAnAccess()).getAnExpr()), kind)
   )
   or
   exists(Function f, ThisExpr innerThis |
@@ -110,7 +112,7 @@ private predicate exprReleases(Expr e, Expr released, string kind) {
     ) and
     e.(FunctionCall).getQualifier() = exprOrDereference(released) and
     innerThis.getEnclosingFunction() = f and
-    exprReleases(_, innerThis, kind)
+    exprReleases(_, globalValueNumber(innerThis).getAnExpr(), kind)
   )
 }