Merge branch 'main' into alamofire2

Author: geoffw0

csharp/ql/consistency-queries/SsaConsistency.ql

@@ -1,23 +1,15 @@
 import csharp
-import semmle.code.csharp.dataflow.internal.SsaImpl::Consistency as Consistency
+import semmle.code.csharp.dataflow.internal.SsaImpl::Consistency
 import Ssa
 
-class MyRelevantDefinition extends Consistency::RelevantDefinition, Ssa::Definition {
+class MyRelevantDefinition extends RelevantDefinition, Ssa::Definition {
   override predicate hasLocationInfo(
     string filepath, int startline, int startcolumn, int endline, int endcolumn
   ) {
     this.getLocation().hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn)
   }
 }
 
-query predicate nonUniqueDef = Consistency::nonUniqueDef/4;
-
-query predicate readWithoutDef = Consistency::readWithoutDef/3;
-
-query predicate deadDef = Consistency::deadDef/2;
-
-query predicate notDominatedByDef = Consistency::notDominatedByDef/4;
-
 query predicate localDeclWithSsaDef(LocalVariableDeclExpr d) {
   // Local variables in C# must be initialized before every use, so uninitialized
   // local variables should not have an SSA definition, as that would imply that

go/extractor/trap/labels.go

@@ -57,7 +57,7 @@ func (l *Labeler) GlobalID(key string) Label {
 	label, exists := l.keyLabels[key]
 	if !exists {
 		id := l.nextID()
-		fmt.Fprintf(l.tw.zip, "%s=@\"%s\"\n", id, escapeString(key))
+		fmt.Fprintf(l.tw.wzip, "%s=@\"%s\"\n", id, escapeString(key))
 		label = Label{id}
 		l.keyLabels[key] = label
 	}
@@ -90,7 +90,7 @@ func (l *Labeler) LocalID(nd interface{}) Label {
 // FreshID creates a fresh label and returns it
 func (l *Labeler) FreshID() Label {
 	id := l.nextID()
-	fmt.Fprintf(l.tw.zip, "%s=*\n", id)
+	fmt.Fprintf(l.tw.wzip, "%s=*\n", id)
 	return Label{id}
 }
 

go/extractor/trap/trapwriter.go

@@ -19,7 +19,8 @@ import (
 // A Writer provides methods for writing data to a TRAP file
 type Writer struct {
 	zip             *gzip.Writer
-	w               *bufio.Writer
+	wzip            *bufio.Writer
+	wfile           *bufio.Writer
 	file            *os.File
 	Labeler         *Labeler
 	path            string
@@ -54,11 +55,13 @@ func NewWriter(path string, pkg *packages.Package) (*Writer, error) {
 	if err != nil {
 		return nil, err
 	}
-	bufioWriter := bufio.NewWriter(tmpFile)
-	zipWriter := gzip.NewWriter(bufioWriter)
+	bufioFileWriter := bufio.NewWriter(tmpFile)
+	zipWriter := gzip.NewWriter(bufioFileWriter)
+	bufioZipWriter := bufio.NewWriter(zipWriter)
 	tw := &Writer{
 		zipWriter,
-		bufioWriter,
+		bufioZipWriter,
+		bufioFileWriter,
 		tmpFile,
 		nil,
 		path,
@@ -88,13 +91,19 @@ func trapFolder() (string, error) {
 
 // Close the underlying file writer
 func (tw *Writer) Close() error {
-	err := tw.zip.Close()
+	err := tw.wzip.Flush()
+	if err != nil {
+		// throw away file close error
+		tw.file.Close()
+		return err
+	}
+	err = tw.zip.Close()
 	if err != nil {
 		// return zip-close error, but ignore file-close error
 		tw.file.Close()
 		return err
 	}
-	err = tw.w.Flush()
+	err = tw.wfile.Flush()
 	if err != nil {
 		// throw away close error because write errors are likely to be more important
 		tw.file.Close()
@@ -145,24 +154,24 @@ func capStringLength(s string) string {
 
 // Emit writes out a tuple of values for the given `table`
 func (tw *Writer) Emit(table string, values []interface{}) error {
-	fmt.Fprintf(tw.zip, "%s(", table)
+	fmt.Fprintf(tw.wzip, "%s(", table)
 	for i, value := range values {
 		if i > 0 {
-			fmt.Fprint(tw.zip, ", ")
+			fmt.Fprint(tw.wzip, ", ")
 		}
 		switch value := value.(type) {
 		case Label:
-			fmt.Fprint(tw.zip, value.id)
+			fmt.Fprint(tw.wzip, value.id)
 		case string:
-			fmt.Fprintf(tw.zip, "\"%s\"", escapeString(capStringLength(value)))
+			fmt.Fprintf(tw.wzip, "\"%s\"", escapeString(capStringLength(value)))
 		case int:
-			fmt.Fprintf(tw.zip, "%d", value)
+			fmt.Fprintf(tw.wzip, "%d", value)
 		case float64:
-			fmt.Fprintf(tw.zip, "%e", value)
+			fmt.Fprintf(tw.wzip, "%e", value)
 		default:
 			return errors.New("Cannot emit value")
 		}
 	}
-	fmt.Fprintf(tw.zip, ")\n")
+	fmt.Fprintf(tw.wzip, ")\n")
 	return nil
 }

java/kotlin-extractor/src/main/kotlin/utils/Logger.kt

@@ -138,6 +138,10 @@ open class LoggerBase(val logCounter: LogCounter) {
         fullMsgBuilder.append(suffix)
 
         val fullMsg = fullMsgBuilder.toString()
+        emitDiagnostic(tw, severity, diagnosticLocStr, msg, fullMsg, locationString, mkLocationId)
+    }
+
+    private fun emitDiagnostic(tw: TrapWriter, severity: Severity, diagnosticLocStr: String, msg: String, fullMsg: String, locationString: String? = null, mkLocationId: () -> Label<DbLocation> = { tw.unknownLocation }) {
         val locStr = if (locationString == null) "" else "At " + locationString + ": "
         val kind = if (severity <= Severity.WarnHigh) "WARN" else "ERROR"
         val logMessage = LogMessage(kind, "Diagnostic($diagnosticLocStr): $locStr$fullMsg")
@@ -190,9 +194,10 @@ open class LoggerBase(val logCounter: LogCounter) {
                 // We don't know if this location relates to an error
                 // or a warning, so we just declare hitting the limit
                 // to be an error regardless.
-                val logMessage = LogMessage("ERROR", "Total of $count diagnostics from $caller.")
-                tw.writeComment(logMessage.toText())
-                logStream.write(logMessage.toJsonLine())
+                val message = "Total of $count diagnostics (reached limit of ${logCounter.diagnosticLimit}) from $caller."
+                if (verbosity >= 1) {
+                    emitDiagnostic(tw, Severity.Error, "Limit", message, message)
+                }
             }
         }
     }

java/ql/src/Telemetry/ExtractorInformation.ql

@@ -9,6 +9,13 @@
 import java
 import semmle.code.java.Diagnostics
 
+predicate compilationInfo(string key, int value) {
+  exists(Compilation c, string infoKey |
+    key = infoKey + ": " + c.getInfo(infoKey) and
+    value = 1
+  )
+}
+
 predicate fileCount(string key, int value) {
   key = "Number of files" and
   value = strictcount(File f)
@@ -53,13 +60,38 @@ predicate extractorDiagnostics(string key, int value) {
   )
 }
 
+/*
+ * Just counting the diagnostics doesn't give the full picture, as
+ * CODEQL_EXTRACTOR_KOTLIN_DIAGNOSTIC_LIMIT means that some diagnostics
+ * will be suppressed. In that case, we need to look for the
+ * suppression message, uncount those that did get emitted, uncount the
+ * suppression message itself, and then add on the full count.
+ */
+
+predicate extractorTotalDiagnostics(string key, int value) {
+  exists(string extractor, string limitRegex |
+    limitRegex = "Total of ([0-9]+) diagnostics \\(reached limit of ([0-9]+)\\).*" and
+    key = "Total number of diagnostics from " + extractor and
+    value =
+      strictcount(Diagnostic d | d.getGeneratedBy() = extractor) +
+        sum(Diagnostic d |
+          d.getGeneratedBy() = extractor
+        |
+          d.getMessage().regexpCapture(limitRegex, 1).toInt() -
+              d.getMessage().regexpCapture(limitRegex, 2).toInt() - 1
+        )
+  )
+}
+
 from string key, int value
 where
+  compilationInfo(key, value) or
   fileCount(key, value) or
   fileCountByExtension(key, value) or
   totalNumberOfLines(key, value) or
   numberOfLinesOfCode(key, value) or
   totalNumberOfLinesByExtension(key, value) or
   numberOfLinesOfCodeByExtension(key, value) or
-  extractorDiagnostics(key, value)
+  extractorDiagnostics(key, value) or
+  extractorTotalDiagnostics(key, value)
 select key, value

javascript/ql/experimental/adaptivethreatmodeling/lib/experimental/adaptivethreatmodeling/ATMConfig.qll

@@ -50,7 +50,8 @@ abstract class AtmConfig extends string {
     // known sink for the class.
     exists(EndpointCharacteristic characteristic |
       characteristic.getEndpoints(sink) and
-      characteristic.getImplications(this.getASinkEndpointType(), true, 1.0)
+      characteristic
+          .getImplications(this.getASinkEndpointType(), true, characteristic.maximalConfidence())
     )
   }