Merge pull request github#12680 from jcogs33/jcogs33/metrics-query-refactor-top500

Java: test GeneratedVsManualCoverage query on top 500 JDK APIs

Author: jcogs33

java/ql/src/Metrics/Summaries/GeneratedVsManualCoverage.ql

@@ -7,71 +7,15 @@
  */
 
 import java
-import semmle.code.java.dataflow.FlowSummary
-import utils.modelgenerator.internal.CaptureModels
-
-/**
- * Returns the number of `DataFlowTargetApi`s with Summary MaD models
- * for a given package and provenance.
- */
-bindingset[package]
-private int getNumMadModeledApis(string package, string provenance) {
-  provenance in ["generated", "manual", "both"] and
-  result =
-    count(SummarizedCallable sc |
-      package = sc.asCallable().getCompilationUnit().getPackage().getName() and
-      sc.asCallable() instanceof DataFlowTargetApi and
-      (
-        // "auto-only"
-        sc.isAutoGenerated() and
-        provenance = "generated"
-        or
-        sc.isManual() and
-        (
-          if sc.hasProvenance(["generated", "ai-generated"])
-          then
-            // "both"
-            provenance = "both"
-          else
-            // "manual-only"
-            provenance = "manual"
-        )
-      )
-    )
-}
-
-/** Returns the total number of `DataFlowTargetApi`s for a given package. */
-private int getNumApis(string package) {
-  result =
-    strictcount(DataFlowTargetApi dataFlowTargApi |
-      package = dataFlowTargApi.getCompilationUnit().getPackage().getName()
-    )
-}
+import GeneratedVsManualCoverageQuery
 
 from
-  string package, int generatedOnly, int both, int manualOnly, int generated, int manual, int non,
-  int all, float coverage, float generatedCoverage, float manualCoverage,
-  float manualCoveredByGenerated, float generatedCoveredByManual, float match
+  string package, int generatedOnly, int both, int manualOnly, int non, int all, float coverage,
+  float generatedCoverage, float manualCoverage, float manualCoveredByGenerated,
+  float generatedCoveredByManual, float match
 where
-  // count the number of APIs with generated-only, both, and manual-only MaD models for each package
-  generatedOnly = getNumMadModeledApis(package, "generated") and
-  both = getNumMadModeledApis(package, "both") and
-  manualOnly = getNumMadModeledApis(package, "manual") and
-  // calculate the total generated and total manual numbers
-  generated = generatedOnly + both and
-  manual = manualOnly + both and
-  // count the total number of `DataFlowTargetApi`s for each package
-  all = getNumApis(package) and
-  non = all - (generatedOnly + both + manualOnly) and
-  // Proportion of coverage
-  coverage = (generatedOnly + both + manualOnly).(float) / all and
-  generatedCoverage = generated.(float) / all and
-  manualCoverage = manual.(float) / all and
-  // Proportion of manual models covered by generated ones
-  manualCoveredByGenerated = both.(float) / (both + manualOnly) and
-  // Proportion of generated models covered by manual ones
-  generatedCoveredByManual = both.(float) / (both + generatedOnly) and
-  // Proportion of data points that match
-  match = (both.(float) + non) / all
+  modelCoverageGenVsMan(package, generatedOnly, both, manualOnly, non, all, coverage,
+    generatedCoverage, manualCoverage, manualCoveredByGenerated, generatedCoveredByManual, match,
+    "allApis")
 select package, generatedOnly, both, manualOnly, non, all, coverage, generatedCoverage,
   manualCoverage, manualCoveredByGenerated, generatedCoveredByManual, match order by package

java/ql/src/Metrics/Summaries/GeneratedVsManualCoverageQuery.qll

@@ -0,0 +1,86 @@
+private import semmle.code.java.dataflow.FlowSummary
+private import utils.modelgenerator.internal.CaptureModels
+private import TopJdkApis
+
+/**
+ * Returns the number of `DataFlowTargetApi`s with Summary MaD models
+ * for a given package and provenance.
+ */
+bindingset[package, apiSubset]
+private int getNumMadModeledApis(string package, string provenance, string apiSubset) {
+  provenance in ["generated", "manual", "both"] and
+  result =
+    count(SummarizedCallable sc |
+      callableSubset(sc.asCallable(), apiSubset) and
+      package = sc.asCallable().getCompilationUnit().getPackage().getName() and
+      sc.asCallable() instanceof DataFlowTargetApi and
+      (
+        // "auto-only"
+        sc.isAutoGenerated() and
+        provenance = "generated"
+        or
+        sc.isManual() and
+        (
+          if sc.hasProvenance(["generated", "ai-generated"])
+          then
+            // "both"
+            provenance = "both"
+          else
+            // "manual-only"
+            provenance = "manual"
+        )
+      )
+    )
+}
+
+/** Returns the total number of `DataFlowTargetApi`s for a given package. */
+private int getNumApis(string package, string apiSubset) {
+  result =
+    strictcount(DataFlowTargetApi dataFlowTargApi |
+      callableSubset(dataFlowTargApi, apiSubset) and
+      package = dataFlowTargApi.getCompilationUnit().getPackage().getName()
+    )
+}
+
+/** Holds if the given `callable` belongs to the specified `apiSubset`. */
+private predicate callableSubset(Callable callable, string apiSubset) {
+  apiSubset in ["topJdkApis", "allApis"] and
+  (
+    if apiSubset = "topJdkApis"
+    then exists(TopJdkApi topJdkApi | callable = topJdkApi.asCallable())
+    else apiSubset = "allApis"
+  )
+}
+
+/**
+ * Provides MaD summary model coverage information for the given `package`
+ * on the given `apiSubset`.
+ */
+predicate modelCoverageGenVsMan(
+  string package, int generatedOnly, int both, int manualOnly, int non, int all, float coverage,
+  float generatedCoverage, float manualCoverage, float manualCoveredByGenerated,
+  float generatedCoveredByManual, float match, string apiSubset
+) {
+  exists(int generated, int manual |
+    // count the number of APIs with generated-only, both, and manual-only MaD models for each package
+    generatedOnly = getNumMadModeledApis(package, "generated", apiSubset) and
+    both = getNumMadModeledApis(package, "both", apiSubset) and
+    manualOnly = getNumMadModeledApis(package, "manual", apiSubset) and
+    // calculate the total generated and total manual numbers
+    generated = generatedOnly + both and
+    manual = manualOnly + both and
+    // count the total number of `DataFlowTargetApi`s for each package
+    all = getNumApis(package, apiSubset) and
+    non = all - (generatedOnly + both + manualOnly) and
+    // Proportion of coverage
+    coverage = (generatedOnly + both + manualOnly).(float) / all and
+    generatedCoverage = generated.(float) / all and
+    manualCoverage = manual.(float) / all and
+    // Proportion of manual models covered by generated ones
+    manualCoveredByGenerated = both.(float) / (both + manualOnly) and
+    // Proportion of generated models covered by manual ones
+    generatedCoveredByManual = both.(float) / (both + generatedOnly) and
+    // Proportion of data points that match
+    match = (both.(float) + non) / all
+  )
+}

java/ql/test/ext/TopJdkApis/TopJdkApis.qll renamed to java/ql/src/Metrics/Summaries/TopJdkApis.qll

@@ -1,5 +1,5 @@
 import java
-import TopJdkApis
+import Metrics.Summaries.TopJdkApis
 
 from string apiName, string message
 where

java/ql/test/ext/TopJdkApis/TopJdkApisTest.ql

@@ -0,0 +1,29 @@
+// Generated automatically from java.awt.Container for testing purposes
+
+package java.awt;
+
+import java.awt.AWTEvent;
+import java.awt.AWTKeyStroke;
+import java.awt.Component;
+import java.awt.ComponentOrientation;
+import java.awt.Dimension;
+import java.awt.Event;
+import java.awt.FocusTraversalPolicy;
+import java.awt.Font;
+import java.awt.Graphics;
+import java.awt.Insets;
+import java.awt.LayoutManager;
+import java.awt.Point;
+import java.awt.event.ContainerEvent;
+import java.awt.event.ContainerListener;
+import java.beans.PropertyChangeListener;
+import java.io.PrintStream;
+import java.io.PrintWriter;
+import java.util.EventListener;
+import java.util.Set;
+
+public class Container extends Component
+{
+    public Component add(Component p0){ return null; } // manual summary
+    public void add(Component p0, Object p1){} // manual summary
+}

java/ql/test/query-tests/Metrics/GeneratedVsManualCoverage/GeneratedVsManualCoverage.expected renamed to java/ql/test/query-tests/Metrics/GeneratedVsManualCoverage/ApacheCommonsIoTest/GeneratedVsManualCoverage.expected

@@ -0,0 +1,11 @@
+// Generated automatically from java.awt.Insets for testing purposes
+
+package java.awt;
+
+import java.io.Serializable;
+
+public class Insets implements Cloneable, Serializable
+{
+    protected Insets() {}
+    public Insets(int p0, int p1, int p2, int p3){} // manual neutral
+}

java/ql/test/query-tests/Metrics/GeneratedVsManualCoverage/GeneratedVsManualCoverage.qlref renamed to java/ql/test/query-tests/Metrics/GeneratedVsManualCoverage/ApacheCommonsIoTest/GeneratedVsManualCoverage.qlref

@@ -0,0 +1,13 @@
+// Generated automatically from java.io.BufferedReader for testing purposes
+
+package java.io;
+
+import java.io.Reader;
+import java.util.stream.Stream;
+
+public class BufferedReader
+{
+    protected BufferedReader() {}
+    public BufferedReader(Reader p0){} // manual summary
+    public String readLine(){ return null; } // manual summary
+}