Merge remote-tracking branch 'origin/main' into criemen/pytest-csharp

criemen · criemen · commit 718d46ed2ff6 · 2024-08-08T09:27:19.000+02:00
diff --git a/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs b/csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/NugetPackageRestorer.cs
@@ -667,7 +667,7 @@ private bool CheckFeeds(out HashSet<string> explicitFeeds)
                     "Found unreachable Nuget feed in C# analysis with build-mode 'none'",
                     visibility: new DiagnosticMessage.TspVisibility(statusPage: true, cliSummaryTable: true, telemetry: true),
                     markdownMessage: "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",
-                    severity: DiagnosticMessage.TspSeverity.Warning
+                    severity: DiagnosticMessage.TspSeverity.Note
                 ));
             }
             compilationInfoContainer.CompilationInfos.Add(("All Nuget feeds reachable", allFeedsReachable ? "1" : "0"));
diff --git a/csharp/ql/integration-tests/all-platforms/standalone/DatabaseQualityDiagnostics.expected b/csharp/ql/integration-tests/all-platforms/standalone/DatabaseQualityDiagnostics.expected
@@ -0,0 +1,6 @@
+diagnosticAttributes
+| Scanning C# code completed successfully, but the scan encountered issues. This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning C# using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes). | visibilityCliSummaryTable | true |
+| Scanning C# code completed successfully, but the scan encountered issues. This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning C# using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes). | visibilityStatusPage | true |
+| Scanning C# code completed successfully, but the scan encountered issues. This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning C# using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes). | visibilityTelemetry | true |
+#select
+| Scanning C# code completed successfully, but the scan encountered issues. This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning C# using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes). | Scanning C# code completed successfully, but the scan encountered issues. This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning C# using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes). | 1 |
diff --git a/csharp/ql/integration-tests/all-platforms/standalone/DatabaseQualityDiagnostics.qlref b/csharp/ql/integration-tests/all-platforms/standalone/DatabaseQualityDiagnostics.qlref
@@ -0,0 +1 @@
+Telemetry/DatabaseQualityDiagnostics.ql
diff --git a/csharp/ql/integration-tests/all-platforms/standalone_buildless_option/DatabaseQualityDiagnostics.expected b/csharp/ql/integration-tests/all-platforms/standalone_buildless_option/DatabaseQualityDiagnostics.expected
@@ -0,0 +1,2 @@
+diagnosticAttributes
+#select
diff --git a/csharp/ql/integration-tests/all-platforms/standalone_buildless_option/DatabaseQualityDiagnostics.qlref b/csharp/ql/integration-tests/all-platforms/standalone_buildless_option/DatabaseQualityDiagnostics.qlref
@@ -0,0 +1 @@
+Telemetry/DatabaseQualityDiagnostics.ql
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/diagnostics.expected b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_error_timeout/diagnostics.expected
@@ -28,7 +28,7 @@
 }
 {
   "markdownMessage": "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",
-  "severity": "warning",
+  "severity": "note",
   "source": {
     "extractorName": "csharp",
     "id": "csharp/autobuilder/buildless/unreachable-feed",
diff --git a/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/diagnostics.expected b/csharp/ql/integration-tests/posix-only/standalone_dependencies_nuget_config_fallback/diagnostics.expected
@@ -28,7 +28,7 @@
 }
 {
   "markdownMessage": "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",
-  "severity": "warning",
+  "severity": "note",
   "source": {
     "extractorName": "csharp",
     "id": "csharp/autobuilder/buildless/unreachable-feed",
diff --git a/csharp/ql/src/Telemetry/DatabaseQuality.qll b/csharp/ql/src/Telemetry/DatabaseQuality.qll
@@ -0,0 +1,72 @@
+/**
+ * Provides database quality statistics that are reported by csharp/telemetry/extractor-information
+ * and perhaps warned about by csharp/diagnostics/database-quality.
+ */
+
+import csharp
+
+signature module StatsSig {
+  int getNumberOfOk();
+
+  int getNumberOfNotOk();
+
+  string getOkText();
+
+  string getNotOkText();
+}
+
+module ReportStats<StatsSig Stats> {
+  predicate numberOfOk(string key, int value) {
+    value = Stats::getNumberOfOk() and
+    key = "Number of " + Stats::getOkText()
+  }
+
+  predicate numberOfNotOk(string key, int value) {
+    value = Stats::getNumberOfNotOk() and
+    key = "Number of " + Stats::getNotOkText()
+  }
+
+  predicate percentageOfOk(string key, float value) {
+    value = Stats::getNumberOfOk() * 100.0 / (Stats::getNumberOfOk() + Stats::getNumberOfNotOk()) and
+    key = "Percentage of " + Stats::getOkText()
+  }
+}
+
+module CallTargetStats implements StatsSig {
+  int getNumberOfOk() { result = count(Call c | exists(c.getTarget())) }
+
+  int getNumberOfNotOk() {
+    result =
+      count(Call c |
+        not exists(c.getTarget()) and
+        not c instanceof DelegateCall and
+        not c instanceof DynamicExpr
+      )
+  }
+
+  string getOkText() { result = "calls with call target" }
+
+  string getNotOkText() { result = "calls with missing call target" }
+}
+
+private class SourceExpr extends Expr {
+  SourceExpr() { this.getFile().fromSource() }
+}
+
+private predicate hasGoodType(Expr e) {
+  exists(e.getType()) and not e.getType() instanceof UnknownType
+}
+
+module ExprTypeStats implements StatsSig {
+  int getNumberOfOk() { result = count(SourceExpr e | hasGoodType(e)) }
+
+  int getNumberOfNotOk() { result = count(SourceExpr e | not hasGoodType(e)) }
+
+  string getOkText() { result = "expressions with known type" }
+
+  string getNotOkText() { result = "expressions with unknown type" }
+}
+
+module CallTargetStatsReport = ReportStats<CallTargetStats>;
+
+module ExprTypeStatsReport = ReportStats<ExprTypeStats>;
diff --git a/csharp/ql/src/Telemetry/DatabaseQualityDiagnostics.ql b/csharp/ql/src/Telemetry/DatabaseQualityDiagnostics.ql
@@ -0,0 +1,44 @@
+/**
+ * @name Low C# analysis quality
+ * @description Low C# analysis quality
+ * @kind diagnostic
+ * @id csharp/diagnostic/database-quality
+ */
+
+import csharp
+import DatabaseQuality
+
+private newtype TDbQualityDiagnostic =
+  TTheDbQualityDiagnostic() {
+    exists(float percentageGood |
+      CallTargetStatsReport::percentageOfOk(_, percentageGood)
+      or
+      ExprTypeStatsReport::percentageOfOk(_, percentageGood)
+    |
+      percentageGood < 95
+    )
+  }
+
+class DbQualityDiagnostic extends TDbQualityDiagnostic {
+  string toString() {
+    result =
+      "Scanning C# code completed successfully, but the scan encountered issues. " +
+        "This may be caused by problems identifying dependencies or use of generated source code, among other reasons -- "
+        +
+        "see other CodeQL diagnostics reported on the CodeQL status page for more details of possible causes. "
+        +
+        "Addressing these warnings is advisable to avoid false-positive or missing results. If they cannot be addressed, consider scanning C# "
+        +
+        "using either the `autobuild` or `manual` [build modes](https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages#comparison-of-the-build-modes)."
+  }
+}
+
+query predicate diagnosticAttributes(DbQualityDiagnostic e, string key, string value) {
+  exists(e) and // Quieten warning about unconstrained 'e'
+  key = ["visibilityCliSummaryTable", "visibilityTelemetry", "visibilityStatusPage"] and
+  value = "true"
+}
+
+from DbQualityDiagnostic d
+select d, d.toString(), 1
+/* Warning severity */
diff --git a/csharp/ql/src/Telemetry/ExtractorInformation.ql b/csharp/ql/src/Telemetry/ExtractorInformation.ql
@@ -8,6 +8,7 @@
 
 import csharp
 import semmle.code.csharp.commons.Diagnostics
+import DatabaseQuality
 
 predicate compilationInfo(string key, float value) {
   not key.matches("Compiler diagnostic count for%") and
@@ -90,68 +91,6 @@ predicate extractionIsStandalone(string key, int value) {
   key = "Is extracted with build-mode set to 'none'"
 }
 
-signature module StatsSig {
-  int getNumberOfOk();
-
-  int getNumberOfNotOk();
-
-  string getOkText();
-
-  string getNotOkText();
-}
-
-module ReportStats<StatsSig Stats> {
-  predicate numberOfOk(string key, int value) {
-    value = Stats::getNumberOfOk() and
-    key = "Number of " + Stats::getOkText()
-  }
-
-  predicate numberOfNotOk(string key, int value) {
-    value = Stats::getNumberOfNotOk() and
-    key = "Number of " + Stats::getNotOkText()
-  }
-
-  predicate percentageOfOk(string key, float value) {
-    value = Stats::getNumberOfOk() * 100.0 / (Stats::getNumberOfOk() + Stats::getNumberOfNotOk()) and
-    key = "Percentage of " + Stats::getOkText()
-  }
-}
-
-module CallTargetStats implements StatsSig {
-  int getNumberOfOk() { result = count(Call c | exists(c.getTarget())) }
-
-  int getNumberOfNotOk() {
-    result =
-      count(Call c |
-        not exists(c.getTarget()) and
-        not c instanceof DelegateCall and
-        not c instanceof DynamicExpr
-      )
-  }
-
-  string getOkText() { result = "calls with call target" }
-
-  string getNotOkText() { result = "calls with missing call target" }
-}
-
-private class SourceExpr extends Expr {
-  SourceExpr() { this.getFile().fromSource() }
-}
-
-private predicate hasGoodType(Expr e) {
-  exists(e.getType()) and not e.getType() instanceof UnknownType
-}
-
-module ExprTypeStats implements StatsSig {
-  int getNumberOfOk() { result = count(SourceExpr e | hasGoodType(e)) }
-
-  int getNumberOfNotOk() { result = count(SourceExpr e | not hasGoodType(e)) }
-
-  string getOkText() { result = "expressions with known type" }
-
-  string getNotOkText() { result = "expressions with unknown type" }
-}
-
 module TypeMentionTypeStats implements StatsSig {
   int getNumberOfOk() { result = count(TypeMention t | not t.getType() instanceof UnknownType) }
 
@@ -182,10 +121,6 @@ module ExprStats implements StatsSig {
   string getNotOkText() { result = "expressions with unknown kind" }
 }
 
-module CallTargetStatsReport = ReportStats<CallTargetStats>;
-
-module ExprTypeStatsReport = ReportStats<ExprTypeStats>;
-
 module TypeMentionTypeStatsReport = ReportStats<TypeMentionTypeStats>;
 
 module AccessTargetStatsReport = ReportStats<AccessTargetStats>;
diff --git a/csharp/ql/src/change-notes/2024-08-07-db-quality-diagnostic.md b/csharp/ql/src/change-notes/2024-08-07-db-quality-diagnostic.md
@@ -0,0 +1,4 @@
+---
+category: minorAnalysis
+---
+* C# build-mode `none` analyses now report a warning on the CodeQL status page when there are significant analysis problems-- defined as 5% of expressions lacking a type, or 5% of call targets being unknown. Other messages reported on the status page are downgraded from warnings to notes and so are less prominent, but are still available for review.
diff --git a/java/ql/src/Telemetry/DatabaseQualityDiagnostics.ql b/java/ql/src/Telemetry/DatabaseQualityDiagnostics.ql
@@ -34,7 +34,7 @@ class DbQualityDiagnostic extends TDbQualityDiagnostic {
 }
 
 query predicate diagnosticAttributes(DbQualityDiagnostic e, string key, string value) {
-  e = e and // Quieten warning about unconstrained 'e'
+  exists(e) and // Quieten warning about unconstrained 'e'
   key = ["visibilityCliSummaryTable", "visibilityTelemetry", "visibilityStatusPage"] and
   value = "true"
 }
diff --git a/javascript/extractor/src/com/semmle/jcorn/Parser.java b/javascript/extractor/src/com/semmle/jcorn/Parser.java
@@ -3547,7 +3547,19 @@ protected List<ExportSpecifier> parseExportSpecifiers(Set<String> exports) {
 
       SourceLocation loc = new SourceLocation(this.startLoc);
       Identifier local = this.parseIdent(this.type == TokenType._default);
-      Identifier exported = this.eatContextual("as") ? this.parseIdent(true) : local;
+      Identifier exported;
+      if (!this.eatContextual("as")) {
+        exported = local;
+      } else {
+        if (this.type == TokenType.string) {
+          // e.g. `export { Foo_new as "Foo::new" }`
+          Expression string = this.parseExprAtom(null);
+          String str = ((Literal)string).getStringValue();
+          exported = this.finishNode(new Identifier(loc, str));
+        } else {
+          exported = this.parseIdent(true);
+        }
+      }
       checkExport(exports, exported.getName(), exported.getLoc().getStart());
       nodes.add(this.finishNode(new ExportSpecifier(loc, local, exported)));
     }
@@ -3629,7 +3641,22 @@ protected List<ImportSpecifier> parseImportSpecifiers() {
 
   protected ImportSpecifier parseImportSpecifier() {
     SourceLocation loc = new SourceLocation(this.startLoc);
-    Identifier imported = this.parseIdent(true), local;
+    Identifier imported, local;
+
+    if (this.type == TokenType.string) {
+      // Arbitrary Module Namespace Identifiers
+      // e.g. `import { "Foo::new" as Foo_new } from "./foo.wasm"`
+      Expression string = this.parseExprAtom(null);    
+      String str = ((Literal)string).getStringValue();
+      imported = this.finishNode(new Identifier(loc, str));
+      // only makes sense if there is a local identifier
+      if (!this.isContextual("as")) {
+        this.raiseRecoverable(this.start, "Unexpected string");
+      }
+    } else {
+      imported = this.parseIdent(true);
+    }
+
     if (this.eatContextual("as")) {
       local = this.parseIdent(false);
     } else {
diff --git a/javascript/extractor/tests/flow/input/arbitaryModuleSpecifier.js b/javascript/extractor/tests/flow/input/arbitaryModuleSpecifier.js
@@ -0,0 +1 @@
+import { "foo" } from "foo"; // syntax-error, but it shouldn't crash the extractor
diff --git a/javascript/extractor/tests/flow/output/trap/arbitaryModuleSpecifier.js.trap b/javascript/extractor/tests/flow/output/trap/arbitaryModuleSpecifier.js.trap
diff --git a/javascript/ql/test/library-tests/Modules/arbitarySpecifier.js b/javascript/ql/test/library-tests/Modules/arbitarySpecifier.js
diff --git a/javascript/ql/test/library-tests/Modules/tests.expected b/javascript/ql/test/library-tests/Modules/tests.expected

Original file line number	Diff line number	Diff line change
`@@ -667,7 +667,7 @@ private bool CheckFeeds(out HashSet<string> explicitFeeds)`
`667`	`667`	`"Found unreachable Nuget feed in C# analysis with build-mode 'none'",`
`668`	`668`	`visibility: new DiagnosticMessage.TspVisibility(statusPage: true, cliSummaryTable: true, telemetry: true),`
`669`	`669`	`markdownMessage: "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",`
`670`		`- severity: DiagnosticMessage.TspSeverity.Warning`
	`670`	`+ severity: DiagnosticMessage.TspSeverity.Note`
`671`	`671`	`));`
`672`	`672`	`}`
`673`	`673`	`compilationInfoContainer.CompilationInfos.Add(("All Nuget feeds reachable", allFeedsReachable ? "1" : "0"));`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+Telemetry/DatabaseQualityDiagnostics.ql`
Original file line number	Diff line number	Diff line change
`@@ -28,7 +28,7 @@`
`28`	`28`	`}`
`29`	`29`	`{`
`30`	`30`	`"markdownMessage": "Found unreachable Nuget feed in C# analysis with build-mode 'none'. This may cause missing dependencies in the analysis.",`
`31`		`- "severity": "warning",`
	`31`	`+ "severity": "note",`
`32`	`32`	`"source": {`
`33`	`33`	`"extractorName": "csharp",`
`34`	`34`	`"id": "csharp/autobuilder/buildless/unreachable-feed",`