Merge pull request github#11585 from jcogs33/jcogs33/mad-metrics-query

Java: add MaD metrics query

Author: jcogs33

csharp/ql/lib/semmle/code/csharp/dataflow/internal/FlowSummaryImpl.qll

@@ -241,9 +241,15 @@ module Public {
     }
 
     /**
-     * Holds if the summary is auto generated.
+     * Holds if the summary is auto generated and not manually generated.
      */
     predicate isAutoGenerated() { none() }
+
+    /**
+     * Holds if the summary has the given provenance where `true` is
+     * `generated` and `false` is `manual`.
+     */
+    predicate hasProvenance(boolean generated) { none() }
   }
 
   /** A callable where there is no flow via the callable. */
@@ -1012,6 +1018,10 @@ module Private {
       }
 
       override predicate isAutoGenerated() { this.relevantSummaryElementGenerated(_, _, _) }
+
+      override predicate hasProvenance(boolean generated) {
+        summaryElement(this, _, _, _, generated)
+      }
     }
 
     /** Holds if component `c` of specification `spec` cannot be parsed. */

go/ql/lib/semmle/go/dataflow/internal/FlowSummaryImpl.qll

@@ -241,9 +241,15 @@ module Public {
     }
 
     /**
-     * Holds if the summary is auto generated.
+     * Holds if the summary is auto generated and not manually generated.
      */
     predicate isAutoGenerated() { none() }
+
+    /**
+     * Holds if the summary has the given provenance where `true` is
+     * `generated` and `false` is `manual`.
+     */
+    predicate hasProvenance(boolean generated) { none() }
   }
 
   /** A callable where there is no flow via the callable. */
@@ -1012,6 +1018,10 @@ module Private {
       }
 
       override predicate isAutoGenerated() { this.relevantSummaryElementGenerated(_, _, _) }
+
+      override predicate hasProvenance(boolean generated) {
+        summaryElement(this, _, _, _, generated)
+      }
     }
 
     /** Holds if component `c` of specification `spec` cannot be parsed. */

java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll

@@ -241,9 +241,15 @@ module Public {
     }
 
     /**
-     * Holds if the summary is auto generated.
+     * Holds if the summary is auto generated and not manually generated.
      */
     predicate isAutoGenerated() { none() }
+
+    /**
+     * Holds if the summary has the given provenance where `true` is
+     * `generated` and `false` is `manual`.
+     */
+    predicate hasProvenance(boolean generated) { none() }
   }
 
   /** A callable where there is no flow via the callable. */
@@ -1012,6 +1018,10 @@ module Private {
       }
 
       override predicate isAutoGenerated() { this.relevantSummaryElementGenerated(_, _, _) }
+
+      override predicate hasProvenance(boolean generated) {
+        summaryElement(this, _, _, _, generated)
+      }
     }
 
     /** Holds if component `c` of specification `spec` cannot be parsed. */

java/ql/src/Metrics/Summaries/GeneratedVsManualCoverage.ql

@@ -0,0 +1,76 @@
+/**
+ * @id java/summary/generated-vs-manual-coverage
+ * @name Metrics of generated versus manual MaD coverage
+ * @description Expose metrics for the number of API endpoints covered by generated versus manual MaD models.
+ * @kind table
+ * @tags summary
+ */
+
+import java
+import semmle.code.java.dataflow.FlowSummary
+import utils.modelgenerator.internal.CaptureModels
+
+/**
+ * Returns the number of `DataFlowTargetApi`s with Summary MaD models
+ * for a given package and provenance.
+ */
+bindingset[package]
+private int getNumMadModeledApis(string package, string provenance) {
+  provenance in ["generated", "manual", "both"] and
+  result =
+    count(SummarizedCallable sc |
+      package = sc.asCallable().getCompilationUnit().getPackage().getName() and
+      sc.asCallable() instanceof DataFlowTargetApi and
+      (
+        // "auto-only"
+        sc.isAutoGenerated() and
+        provenance = "generated"
+        or
+        // "manual-only"
+        sc.hasProvenance(false) and
+        not sc.hasProvenance(true) and
+        provenance = "manual"
+        or
+        // "both"
+        sc.hasProvenance(false) and
+        sc.hasProvenance(true) and
+        provenance = "both"
+      )
+    )
+}
+
+/** Returns the total number of `DataFlowTargetApi`s for a given package. */
+private int getNumApis(string package) {
+  result =
+    strictcount(DataFlowTargetApi dataFlowTargApi |
+      package = dataFlowTargApi.getCompilationUnit().getPackage().getName()
+    )
+}
+
+from
+  string package, int generatedOnly, int both, int manualOnly, int generated, int manual, int non,
+  int all, float coverage, float generatedCoverage, float manualCoverage,
+  float manualCoveredByGenerated, float generatedCoveredByManual, float match
+where
+  // count the number of APIs with generated-only, both, and manual-only MaD models for each package
+  generatedOnly = getNumMadModeledApis(package, "generated") and
+  both = getNumMadModeledApis(package, "both") and
+  manualOnly = getNumMadModeledApis(package, "manual") and
+  // calculate the total generated and total manual numbers
+  generated = generatedOnly + both and
+  manual = manualOnly + both and
+  // count the total number of `DataFlowTargetApi`s for each package
+  all = getNumApis(package) and
+  non = all - (generatedOnly + both + manualOnly) and
+  // Proportion of coverage
+  coverage = (generatedOnly + both + manualOnly).(float) / all and
+  generatedCoverage = generated.(float) / all and
+  manualCoverage = manual.(float) / all and
+  // Proportion of manual models covered by generated ones
+  manualCoveredByGenerated = both.(float) / (both + manualOnly) and
+  // Proportion of generated models covered by manual ones
+  generatedCoveredByManual = both.(float) / (both + generatedOnly) and
+  // Proportion of data points that match
+  match = (both.(float) + non) / all
+select package, generatedOnly, both, manualOnly, non, all, coverage, generatedCoverage,
+  manualCoverage, manualCoveredByGenerated, generatedCoveredByManual, match order by package

java/ql/src/change-notes/2022-12-12-java-mad-metrics-query.md

@@ -0,0 +1,4 @@
+---
+category: newQuery
+---
+* Added a new query, `java/summary/generated-vs-manual-coverage`, to expose metrics for the number of API endpoints covered by generated versus manual MaD models.

java/ql/test/query-tests/Metrics/GeneratedVsManualCoverage/GeneratedVsManualCoverage.expected

@@ -0,0 +1 @@
+| org.apache.commons.io | 14 | 1 | 1 | 2 | 18 | 0.8888888888888888 | 0.8333333333333334 | 0.1111111111111111 | 0.5 | 0.06666666666666667 | 0.16666666666666666 |

java/ql/test/query-tests/Metrics/GeneratedVsManualCoverage/GeneratedVsManualCoverage.qlref

@@ -0,0 +1 @@
+Metrics/Summaries/GeneratedVsManualCoverage.ql

java/ql/test/query-tests/Metrics/GeneratedVsManualCoverage/IOUtils.java

@@ -0,0 +1,34 @@
+package org.apache.commons.io;
+import java.io.*;
+import java.util.*;
+
+public class IOUtils {
+
+  // Generated-only summaries
+  public static BufferedInputStream buffer(InputStream inputStream) { return null; }
+  public static void copy(InputStream input, Writer output, String inputEncoding) throws IOException { }
+  public static long copyLarge(Reader input, Writer output) throws IOException { return 42; }
+  public static int read(InputStream input, byte[] buffer) throws IOException { return 42; }
+  public static void readFully(InputStream input, byte[] buffer) throws IOException { }
+  public static byte[] readFully(InputStream input, int length) throws IOException { return null; }
+  public static List<String> readLines(InputStream input, String encoding) throws IOException { return null; }
+  public static BufferedReader toBufferedReader(Reader reader) { return null; }
+  public static byte[] toByteArray(InputStream input, int size) throws IOException { return null; }
+  public static char[] toCharArray(InputStream is, String encoding) throws IOException { return null; }
+  public static InputStream toInputStream(String input, String encoding) throws IOException { return null; }
+  public static String toString(InputStream input, String encoding) throws IOException { return null; }
+  public static void write(char[] data, Writer output) throws IOException { }
+  public static void writeChunked(char[] data, Writer output) throws IOException { }
+
+  // Manual-only summary (generated neutral)
+  public static InputStream toBufferedInputStream(InputStream input) throws IOException { return null; }
+
+  // Both
+  public static void writeLines(Collection<?> lines, String lineEnding, Writer writer) throws IOException { }
+
+  // No model
+  public static void noSummary(String string) throws IOException { }
+
+  // Generated neutral
+  public static void copy(Reader input, OutputStream output) throws IOException { }
+}