Merge pull request #17169 from tamasvajk/buildless/db-quality-query

tamasvajk · web-flow · commit ef21ee53a563 · 2024-08-07T21:36:37.000+02:00
C#: Add diagnostic query indicating low database quality
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs
@@ -667,7 +667,7 @@ private bool CheckFeeds(out HashSet<string> explicitFeeds)
                     "Found unreachable Nuget feed in C# analysis with build-mode 'none'",
                     visibility: new DiagnosticMessage.TspVisibility(statusPage: true, cliSummaryTable: true, telemetry: true),
                     markdownMessage: "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",
-                    severity: DiagnosticMessage.TspSeverity.Warning
+                    severity: DiagnosticMessage.TspSeverity.Note
                 ));
             }
             compilationInfoContainer.CompilationInfos.Add(("All Nuget feeds reachable", allFeedsReachable ? "1" : "0"));
diff --git a/csharp/ql/integration-tests/all-platforms/standalone/DatabaseQualityDiagnostics.expected b/csharp/ql/integration-tests/all-platforms/standalone/DatabaseQualityDiagnostics.expected
@@ -0,0 +1,6 @@
+diagnosticAttributes
+| Scanning C# code completed successfully, but the scan encountered issues. This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning C# using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes). | visibilityCliSummaryTable | true |
+| Scanning C# code completed successfully, but the scan encountered issues. This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning C# using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes). | visibilityStatusPage | true |
+| Scanning C# code completed successfully, but the scan encountered issues. This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning C# using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes). | visibilityTelemetry | true |
+#select
+| Scanning C# code completed successfully, but the scan encountered issues. This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning C# using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes). | Scanning C# code completed successfully, but the scan encountered issues. This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning C# using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes). | 1 |
diff --git a/csharp/ql/integration-tests/all-platforms/standalone/DatabaseQualityDiagnostics.qlref b/csharp/ql/integration-tests/all-platforms/standalone/DatabaseQualityDiagnostics.qlref
@@ -0,0 +1 @@
+Telemetry/DatabaseQualityDiagnostics.ql
diff --git a/csharp/ql/integration-tests/all-platforms/standalone_buildless_option/DatabaseQualityDiagnostics.expected b/csharp/ql/integration-tests/all-platforms/standalone_buildless_option/DatabaseQualityDiagnostics.expected
@@ -0,0 +1,2 @@
+diagnosticAttributes
+#select
diff --git a/csharp/ql/integration-tests/all-platforms/standalone_buildless_option/DatabaseQualityDiagnostics.qlref b/csharp/ql/integration-tests/all-platforms/standalone_buildless_option/DatabaseQualityDiagnostics.qlref
@@ -0,0 +1 @@
+Telemetry/DatabaseQualityDiagnostics.ql
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/diagnostics.expected b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/diagnostics.expected
@@ -28,7 +28,7 @@
 }
 {
   "markdownMessage": "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",
-  "severity": "warning",
+  "severity": "note",
   "source": {
     "extractorName": "csharp",
     "id": "csharp/autobuilder/buildless/unreachable-feed",
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/diagnostics.expected b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/diagnostics.expected
@@ -28,7 +28,7 @@
 }
 {
   "markdownMessage": "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",
-  "severity": "warning",
+  "severity": "note",
   "source": {
     "extractorName": "csharp",
     "id": "csharp/autobuilder/buildless/unreachable-feed",
diff --git a/csharp/ql/integration-tests/qlpack.yml b/csharp/ql/integration-tests/qlpack.yml
@@ -1,3 +1,4 @@
 dependencies:
   codeql/csharp-all: '*'
+  codeql/csharp-queries: '*'
 warnOnImplicitThis: true
diff --git a/csharp/ql/src/Telemetry/DatabaseQuality.qll b/csharp/ql/src/Telemetry/DatabaseQuality.qll
@@ -0,0 +1,72 @@
+/**
+ * Provides database quality statistics that are reported by csharp/telemetry/extractor-information
+ * and perhaps warned about by csharp/diagnostics/database-quality.
+ */
+
+import csharp
+
+signature module StatsSig {
+  int getNumberOfOk();
+
+  int getNumberOfNotOk();
+
+  string getOkText();
+
+  string getNotOkText();
+}
+
+module ReportStats<StatsSig Stats> {
+  predicate numberOfOk(string key, int value) {
+    value = Stats::getNumberOfOk() and
+    key = "Number of " + Stats::getOkText()
+  }
+
+  predicate numberOfNotOk(string key, int value) {
+    value = Stats::getNumberOfNotOk() and
+    key = "Number of " + Stats::getNotOkText()
+  }
+
+  predicate percentageOfOk(string key, float value) {
+    value = Stats::getNumberOfOk() * 100.0 / (Stats::getNumberOfOk() + Stats::getNumberOfNotOk()) and
+    key = "Percentage of " + Stats::getOkText()
+  }
+}
+
+module CallTargetStats implements StatsSig {
+  int getNumberOfOk() { result = count(Call c | exists(c.getTarget())) }
+
+  int getNumberOfNotOk() {
+    result =
+      count(Call c |
+        not exists(c.getTarget()) and
+        not c instanceof DelegateCall and
+        not c instanceof DynamicExpr
+      )
+  }
+
+  string getOkText() { result = "calls with call target" }
+
+  string getNotOkText() { result = "calls with missing call target" }
+}
+
+private class SourceExpr extends Expr {
+  SourceExpr() { this.getFile().fromSource() }
+}
+
+private predicate hasGoodType(Expr e) {
+  exists(e.getType()) and not e.getType() instanceof UnknownType
+}
+
+module ExprTypeStats implements StatsSig {
+  int getNumberOfOk() { result = count(SourceExpr e | hasGoodType(e)) }
+
+  int getNumberOfNotOk() { result = count(SourceExpr e | not hasGoodType(e)) }
+
+  string getOkText() { result = "expressions with known type" }
+
+  string getNotOkText() { result = "expressions with unknown type" }
+}
+
+module CallTargetStatsReport = ReportStats<CallTargetStats>;
+
+module ExprTypeStatsReport = ReportStats<ExprTypeStats>;
diff --git a/csharp/ql/src/Telemetry/DatabaseQualityDiagnostics.ql b/csharp/ql/src/Telemetry/DatabaseQualityDiagnostics.ql
@@ -0,0 +1,44 @@
+/**
+ * @name Low C# analysis quality
+ * @description Low C# analysis quality
+ * @kind diagnostic
+ * @id csharp/diagnostic/database-quality
+ */
+
+import csharp
+import DatabaseQuality
+
+private newtype TDbQualityDiagnostic =
+  TTheDbQualityDiagnostic() {
+    exists(float percentageGood |
+      CallTargetStatsReport::percentageOfOk(_, percentageGood)
+      or
+      ExprTypeStatsReport::percentageOfOk(_, percentageGood)
+    |
+      percentageGood < 95
+    )
+  }
+
+class DbQualityDiagnostic extends TDbQualityDiagnostic {
+  string toString() {
+    result =
+      "Scanning C# code completed successfully, but the scan encountered issues. " +
+        "This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- "
+        +
+        "see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. "
+        +
+        "Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning C# "
+        +
+        "using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes)."
+  }
+}
+
+query predicate diagnosticAttributes(DbQualityDiagnostic e, string key, string value) {
+  exists(e) and // Quieten warning about unconstrained 'e'
+  key = ["visibilityCliSummaryTable", "visibilityTelemetry", "visibilityStatusPage"] and
+  value = "true"
+}
+
+from DbQualityDiagnostic d
+select d, d.toString(), 1
+/* Warning severity */
diff --git a/csharp/ql/src/Telemetry/ExtractorInformation.ql b/csharp/ql/src/Telemetry/ExtractorInformation.ql
@@ -8,6 +8,7 @@
 
 import csharp
 import semmle.code.csharp.commons.Diagnostics
+import DatabaseQuality
 
 predicate compilationInfo(string key, float value) {
   not key.matches("Compiler diagnostic count for%") and
@@ -90,68 +91,6 @@ predicate extractionIsStandalone(string key, int value) {
   key = "Is extracted with build-mode set to 'none'"
 }
 
-signature module StatsSig {
-  int getNumberOfOk();
-
-  int getNumberOfNotOk();
-
-  string getOkText();
-
-  string getNotOkText();
-}
-
-module ReportStats<StatsSig Stats> {
-  predicate numberOfOk(string key, int value) {
-    value = Stats::getNumberOfOk() and
-    key = "Number of " + Stats::getOkText()
-  }
-
-  predicate numberOfNotOk(string key, int value) {
-    value = Stats::getNumberOfNotOk() and
-    key = "Number of " + Stats::getNotOkText()
-  }
-
-  predicate percentageOfOk(string key, float value) {
-    value = Stats::getNumberOfOk() * 100.0 / (Stats::getNumberOfOk() + Stats::getNumberOfNotOk()) and
-    key = "Percentage of " + Stats::getOkText()
-  }
-}
-
-module CallTargetStats implements StatsSig {
-  int getNumberOfOk() { result = count(Call c | exists(c.getTarget())) }
-
-  int getNumberOfNotOk() {
-    result =
-      count(Call c |
-        not exists(c.getTarget()) and
-        not c instanceof DelegateCall and
-        not c instanceof DynamicExpr
-      )
-  }
-
-  string getOkText() { result = "calls with call target" }
-
-  string getNotOkText() { result = "calls with missing call target" }
-}
-
-private class SourceExpr extends Expr {
-  SourceExpr() { this.getFile().fromSource() }
-}
-
-private predicate hasGoodType(Expr e) {
-  exists(e.getType()) and not e.getType() instanceof UnknownType
-}
-
-module ExprTypeStats implements StatsSig {
-  int getNumberOfOk() { result = count(SourceExpr e | hasGoodType(e)) }
-
-  int getNumberOfNotOk() { result = count(SourceExpr e | not hasGoodType(e)) }
-
-  string getOkText() { result = "expressions with known type" }
-
-  string getNotOkText() { result = "expressions with unknown type" }
-}
-
 module TypeMentionTypeStats implements StatsSig {
   int getNumberOfOk() { result = count(TypeMention t | not t.getType() instanceof UnknownType) }
 
@@ -182,10 +121,6 @@ module ExprStats implements StatsSig {
   string getNotOkText() { result = "expressions with unknown kind" }
 }
 
-module CallTargetStatsReport = ReportStats<CallTargetStats>;
-
-module ExprTypeStatsReport = ReportStats<ExprTypeStats>;
-
 module TypeMentionTypeStatsReport = ReportStats<TypeMentionTypeStats>;
 
 module AccessTargetStatsReport = ReportStats<AccessTargetStats>;
diff --git a/csharp/ql/src/change-notes/2024-08-07-db-quality-diagnostic.md b/csharp/ql/src/change-notes/2024-08-07-db-quality-diagnostic.md
@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* C# build-mode `none` analyses now report a warning on the CodeQL status page when there are significant analysis problems-- defined as 5% of expressions lacking a type, or 5% of call targets being unknown. Other messages reported on the status page are downgraded from warnings to notes and so are less prominent, but are still available for review.
diff --git a/java/ql/src/Telemetry/DatabaseQualityDiagnostics.ql b/java/ql/src/Telemetry/DatabaseQualityDiagnostics.ql
@@ -34,7 +34,7 @@ class DbQualityDiagnostic extends TDbQualityDiagnostic {
 }
 
 query predicate diagnosticAttributes(DbQualityDiagnostic e, string key, string value) {
-  e = e and // Quieten warning about unconstrained 'e'
+  exists(e) and // Quieten warning about unconstrained 'e'
   key = ["visibilityCliSummaryTable", "visibilityTelemetry", "visibilityStatusPage"] and
   value = "true"
 }

Original file line number	Diff line number	Diff line change
`@@ -667,7 +667,7 @@ private bool CheckFeeds(out HashSet<string> explicitFeeds)`
`667`	`667`	`"Found unreachable Nuget feed in C# analysis with build-mode 'none'",`
`668`	`668`	`visibility: new DiagnosticMessage.TspVisibility(statusPage: true, cliSummaryTable: true, telemetry: true),`
`669`	`669`	`markdownMessage: "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",`
`670`		`- severity: DiagnosticMessage.TspSeverity.Warning`
	`670`	`+ severity: DiagnosticMessage.TspSeverity.Note`
`671`	`671`	`));`
`672`	`672`	`}`
`673`	`673`	`compilationInfoContainer.CompilationInfos.Add(("All Nuget feeds reachable", allFeedsReachable ? "1" : "0"));`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+Telemetry/DatabaseQualityDiagnostics.ql`
Original file line number	Diff line number	Diff line change
`@@ -28,7 +28,7 @@`
`28`	`28`	`}`
`29`	`29`	`{`
`30`	`30`	`"markdownMessage": "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",`
`31`		`- "severity": "warning",`
	`31`	`+ "severity": "note",`
`32`	`32`	`"source": {`
`33`	`33`	`"extractorName": "csharp",`
`34`	`34`	`"id": "csharp/autobuilder/buildless/unreachable-feed",`