Python: Add basic call-graph metric queries

For use with dist-compare

Author: RasmusWL

python/ql/src/meta/MetaMetrics.qll

@@ -0,0 +1,27 @@
+/**
+ * Helpers to generating meta metrics, that is, metrics about the CodeQL analysis and extractor.
+ */
+
+import python
+
+private import semmle.python.filters.GeneratedCode
+private import semmle.python.filters.Tests
+
+/**
+ * Gets the root folder of the snapshot.
+ *
+ * This is selected as the location for project-wide metrics.
+ */
+Folder projectRoot() { result.getRelativePath() = "" }
+
+/** A file we ignore because it is a test file, part of a third-part library, or compiled/generated/bundled code. */
+class IgnoredFile extends File {
+  IgnoredFile() {
+    any(TestScope ts).getLocation().getFile() = this
+    or
+    this instanceof GeneratedFile
+    or
+    // outside source root (inspired by `Scope.inSource`)
+    not exists(this.getRelativePath())
+  }
+}

python/ql/src/meta/analysis-quality/CallGraphQuality.qll

@@ -0,0 +1,65 @@
+/**
+ * Provides predicates for measuring the quality of the call graph, that is,
+ * the number of calls that could be resolved to a callee.
+ */
+
+import python
+import meta.MetaMetrics
+
+/**
+ * A call that is (possibly) relevant for analysis quality.
+ * See `IgnoredFile` for details on what is excluded.
+ */
+class RelevantCall extends Call {
+  RelevantCall() { not this.getLocation().getFile() instanceof IgnoredFile }
+}
+
+/** Provides classes for call-graph resolution by using points-to */
+module PointsTo {
+  /** A call that can be resolved by points-to. */
+  class ResolvableCall extends RelevantCall {
+    Value target;
+
+    ResolvableCall() { target.getACall() = this.getAFlowNode() }
+
+    /** Gets a resolved target of this call */
+    Value getTarget() { result = target }
+  }
+
+  /** A call that cannot be resolved by points-to. */
+  class UnresolvableCall extends RelevantCall {
+    UnresolvableCall() { not this instanceof ResolvableCall }
+  }
+
+  /**
+   * A call that can be resolved by points-to, where the resolved target is relevant.
+   * Relevant targets include:
+   * - builtins
+   * - standard library
+   * - source code of the project
+   */
+  class ResolvableCallRelevantTarget extends ResolvableCall {
+    ResolvableCallRelevantTarget() {
+      target.isBuiltin()
+      or
+      exists(File file |
+        file = target.(CallableValue).getScope().getLocation().getFile()
+        or
+        file = target.(ClassValue).getScope().getLocation().getFile()
+      |
+        file.inStdlib()
+        or
+        // part of the source code of the project
+        exists(file.getRelativePath())
+      )
+    }
+  }
+
+  /**
+   * A call that can be resolved by points-to, where resolved target is not considered relevant.
+   * See `ResolvableCallRelevantTarget` for definition of relevance.
+   */
+  class ResolvableCallIrrelevantTarget extends ResolvableCall {
+    ResolvableCallIrrelevantTarget() { not this instanceof ResolvableCallRelevantTarget }
+  }
+}

python/ql/src/meta/analysis-quality/PointsToResolvableCallRatio.ql

@@ -0,0 +1,14 @@
+/**
+ * @name Ratio of resolvable call by points-to
+ * @description The percentage (relevant) calls that can be resolved to a target.
+ * @kind metric
+ * @metricType project
+ * @metricAggregate sum min max avg
+ * @tags meta
+ * @id py/meta/points-to-resolvable-call-ratio
+ */
+
+import python
+import CallGraphQuality
+
+select projectRoot(), 100.0 * count(PointsTo::ResolvableCall call) / count(RelevantCall call).(float)

python/ql/src/meta/analysis-quality/PointsToResolvableCalls.ql

@@ -0,0 +1,14 @@
+/**
+ * @name Resolvable calls by points-to
+ * @description The number of (relevant) calls that could be resolved to its target.
+ * @kind metric
+ * @metricType project
+ * @metricAggregate sum
+ * @tags meta
+ * @id py/meta/points-to-resolvable-calls
+ */
+
+import python
+import CallGraphQuality
+
+select projectRoot(), count(PointsTo::ResolvableCall call)

python/ql/src/meta/analysis-quality/PointsToResolvableCallsRelevantTarget.ql

@@ -0,0 +1,14 @@
+/**
+ * @name Resolvable calls by points-to, to relevant target
+ * @description The number of (relevant) calls that could be resolved to its target that is relevant.
+ * @kind metric
+ * @metricType project
+ * @metricAggregate sum
+ * @tags meta
+ * @id py/meta/points-to-resolvable-calls-relevant-target
+ */
+
+import python
+import CallGraphQuality
+
+select projectRoot(), count(PointsTo::ResolvableCallRelevantTarget call)

python/ql/src/meta/analysis-quality/ResolvableCallCandidates.ql

@@ -0,0 +1,14 @@
+/**
+ * @name Resolvable call candidates
+ * @description The number (relevant) calls in the program.
+ * @kind metric
+ * @metricType project
+ * @metricAggregate sum
+ * @tags meta
+ * @id py/meta/resolvable-call-candidates
+ */
+
+import python
+import CallGraphQuality
+
+select projectRoot(), count(RelevantCall call)