Merge pull request github#3819 from dbartol/codeql-c-analysis-team/40/2

C++: More IR QLDoc (including `Opcode.qll`)

Author: jbj

config/opcode-qldoc.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+
+import os
+import re
+path = os.path
+
+needs_an_re = re.compile(r'^(?!Unary)[AEIOU]')  # Name requiring "an" instead of "a".
+start_qldoc_re = re.compile(r'^\s*/\*\*')  # Start of a QLDoc comment
+end_qldoc_re = re.compile(r'\*/\s*$')  # End of a QLDoc comment
+blank_qldoc_line_re = re.compile(r'^\s*\*\s*$')  # A line in a QLDoc comment with only the '*'
+instruction_class_re = re.compile(r'^class (?P<name>[A-aa-z0-9]+)Instruction\s')  # Declaration of an `Instruction` class
+opcode_base_class_re = re.compile(r'^abstract class (?P<name>[A-aa-z0-9]+)Opcode\s')  # Declaration of an `Opcode` base class
+opcode_class_re = re.compile(r'^  class (?P<name>[A-aa-z0-9]+)\s')  # Declaration of an `Opcode` class
+
+script_dir = path.realpath(path.dirname(__file__))
+instruction_path = path.realpath(path.join(script_dir, '../cpp/ql/src/semmle/code/cpp/ir/implementation/raw/Instruction.qll'))
+opcode_path = path.realpath(path.join(script_dir, '../cpp/ql/src/semmle/code/cpp/ir/implementation/Opcode.qll'))
+
+# Scan `Instruction.qll`, keeping track of the QLDoc comment attached to each declaration of a class
+# whose name ends with `Instruction`.
+instruction_comments = {}
+in_qldoc = False
+saw_blank_line_in_qldoc = False
+qldoc_lines = []
+with open(instruction_path, 'r', encoding='utf-8') as instr:
+    for line in instr:
+        if in_qldoc:
+            if end_qldoc_re.search(line):
+                qldoc_lines.append(line)
+                in_qldoc = False
+            elif blank_qldoc_line_re.search(line):
+                # We're going to skip any lines after the first blank line, to avoid duplicating all
+                # of the verbose description.
+                saw_blank_line_in_qldoc = True
+            elif not saw_blank_line_in_qldoc:
+                qldoc_lines.append(line)
+        else:
+            if start_qldoc_re.search(line):
+                # Starting a new QLDoc comment.
+                saw_blank_line_in_qldoc = False
+                qldoc_lines.append(line)
+                if not end_qldoc_re.search(line):
+                    in_qldoc = True
+            else:
+                instruction_match = instruction_class_re.search(line)
+                if instruction_match:
+                    # Found the declaration of an `Instruction` class. Record the QLDoc comments.
+                    instruction_comments[instruction_match.group('name')] = qldoc_lines
+                qldoc_lines = []
+
+# Scan `Opcode.qll`. Whenever we see the declaration of an `Opcode` class for which we have a
+# corresponding `Instruction` class, we'll attach a copy of the `Instruction`'s QLDoc comment.
+in_qldoc = False
+qldoc_lines = []
+output_lines = []
+with open(opcode_path, 'r', encoding='utf-8') as opcode:
+    for line in opcode:
+        if in_qldoc:
+            qldoc_lines.append(line)
+            if end_qldoc_re.search(line):
+                in_qldoc = False
+        else:
+            if start_qldoc_re.search(line):
+                qldoc_lines.append(line)
+                if not end_qldoc_re.search(line):
+                    in_qldoc = True
+            else:
+                name_without_suffix = None
+                name = None
+                indent = ''
+                opcode_base_match = opcode_base_class_re.search(line)
+                if opcode_base_match:
+                    name_without_suffix = opcode_base_match.group('name')
+                    name = name_without_suffix + 'Opcode'
+                else:
+                    opcode_match = opcode_class_re.search(line)
+                    if opcode_match:
+                        name_without_suffix = opcode_match.group('name')
+                        name = name_without_suffix
+                        # Indent by two additional spaces, since opcodes are declared in the
+                        # `Opcode` module.
+                        indent = '  '
+                
+                if name_without_suffix:
+                    # Found an `Opcode` that matches a known `Instruction`. Replace the QLDoc with
+                    # a copy of the one from the `Instruction`.
+                    if instruction_comments.get(name_without_suffix):
+                        article = 'an' if needs_an_re.search(name_without_suffix) else 'a'
+                        qldoc_lines = [
+                            indent + '/**\n',
+                            indent + ' * The `Opcode` for ' + article + ' `' + name_without_suffix + 'Instruction`.\n',
+                            indent + ' *\n',
+                            indent + ' * See the `' + name_without_suffix + 'Instruction` documentation for more details.\n',
+                            indent + ' */\n'
+                        ]
+                output_lines.extend(qldoc_lines)
+                qldoc_lines = []
+                output_lines.append(line)
+
+# Write out the updated `Opcode.qll`
+with open(opcode_path, 'w', encoding='utf-8') as opcode:
+    opcode.writelines(output_lines)

cpp/ql/src/semmle/code/cpp/ir/IR.qll

@@ -1,3 +1,47 @@
+/**
+ * Provides classes that describe the Intermediate Representation (IR) of the program.
+ *
+ * The IR is a representation of the semantics of the program, with very little dependence on the
+ * syntax that was used to write the program. For example, in C++, the statements `i += 1;`, `i++`,
+ * and `++i` all have the same semantic effect, but appear in the AST as three different types of
+ * `Expr` node. In the IR, all three statements are broken down into a sequence of fundamental
+ * operations similar to:
+ *
+ * ```
+ * r1(int*) = VariableAddress[i]  // Compute the address of variable `i`
+ * r2(int) = Load &:r1, m0  // Load the value of `i`
+ * r3(int) = Constant[1]  // An integer constant with the value `1`
+ * r4(int) = Add r2, r3  // Add `1` to the value of `i`
+ * r5(int) = Store &r1, r4  // Store the new value back into the variable `i`
+ * ```
+ *
+ * This allows IR-based analysis to focus on the fundamental operations, rather than having to be
+ * concerned with the various ways of expressing those operations in source code.
+ *
+ * The key classes in the IR are:
+ *
+ * - `IRFunction` - Contains the IR for an entire function definition, including all of that
+ *   function's `Instruction`s, `IRBlock`s, and `IRVariables`.
+ * - `Instruction` - A single operation in the IR. An instruction specifies the operation to be
+ *   performed, the operands that produce the inputs to that operation, and the type of the result
+ *   of the operation. Control flows from an `Instruction` to one of a set of successor
+ *   `Instruction`s.
+ * - `Operand` - An input value of an `Instruction`. All inputs of an `Instruction` are explicitly
+ *   represented as `Operand`s, even if the input was implicit in the source code. An `Operand` has
+ *   a link to the `Instruction` that consumes its value (its "use") and a link to the `Instruction`
+ *   that produces its value (its "definition").
+ * - `IRVariable` - A variable accessed by the IR for a particular function. An `IRVariable` is
+ *   created for each variable directly accessed by the function. In addition, `IRVariable`s are
+ *   created to represent certain temporary storage locations that do not have explicitly declared
+ *   variables in the source code, such as the return value of the function.
+ * - `IRBlock` - A "basic block" in the control flow graph of a function. An `IRBlock` contains a
+ *   sequence of instructions such that control flow can only enter the block at the first
+ *   instruction, and can only leave the block from the last instruction.
+ * - `IRType` - The type of a value accessed in the IR. Unlike the `Type` class in the AST, `IRType`
+ *   is language-neutral. For example, in C++, `unsigned int`, `char32_t`, and `wchar_t` might all
+ *   be represented as the `IRType` `uint4`, a four-byte unsigned integer.
+ */
+
 // Most queries should operate on the aliased SSA IR, so that's what we expose
-// publically as the "IR".
+// publicly as the "IR".
 import implementation.aliased_ssa.IR

cpp/ql/src/semmle/code/cpp/ir/IRConfiguration.qll

@@ -1 +1,5 @@
+/**
+ * Module used to configure the IR generation process.
+ */
+
 import implementation.IRConfiguration

cpp/ql/src/semmle/code/cpp/ir/PrintIR.qll

@@ -1 +1,11 @@
+/**
+ * Outputs a representation of the IR as a control flow graph.
+ *
+ * This file contains the actual implementation of `PrintIR.ql`. For test cases and very small
+ * databases, `PrintIR.ql` can be run directly to dump the IR for the entire database. For most
+ * uses, however, it is better to write a query that imports `PrintIR.qll`, extends
+ * `PrintIRConfiguration`, and overrides `shouldPrintFunction()` to select a subset of functions to
+ * dump.
+ */
+
 import implementation.aliased_ssa.PrintIR

cpp/ql/src/semmle/code/cpp/ir/implementation/EdgeKind.qll

@@ -1,3 +1,7 @@
+/**
+ * Provides classes that specify the conditions under which control flows along a given edge.
+ */
+
 private import internal.EdgeKindInternal
 
 private newtype TEdgeKind =
@@ -77,9 +81,15 @@ class CaseEdge extends EdgeKind, TCaseEdge {
     else result = "Case[" + minValue + ".." + maxValue + "]"
   }
 
-  string getMinValue() { result = minValue }
+  /**
+   * Gets the smallest value of the switch expression for which control will flow along this edge.
+   */
+  final string getMinValue() { result = minValue }
 
-  string getMaxValue() { result = maxValue }
+  /**
+   * Gets the largest value of the switch expression for which control will flow along this edge.
+   */
+  final string getMaxValue() { result = maxValue }
 }
 
 /**

cpp/ql/src/semmle/code/cpp/ir/implementation/IRConfiguration.qll

@@ -10,13 +10,21 @@ private newtype TIRConfiguration = MkIRConfiguration()
  * The query can extend this class to control which functions have IR generated for them.
  */
 class IRConfiguration extends TIRConfiguration {
+  /** Gets a textual representation of this element. */
   string toString() { result = "IRConfiguration" }
 
   /**
    * Holds if IR should be created for function `func`. By default, holds for all functions.
    */
   predicate shouldCreateIRForFunction(Language::Function func) { any() }
 
+  /**
+   * Holds if the strings used as part of an IR dump should be generated for function `func`.
+   *
+   * This predicate is overridden in `PrintIR.qll` to avoid the expense of generating a large number
+   * of debug strings for IR that will not be dumped. We still generate the actual IR for these
+   * functions, however, to preserve the results of any interprocedural analysis.
+   */
   predicate shouldEvaluateDebugStringsForFunction(Language::Function func) { any() }
 }
 
@@ -26,6 +34,7 @@ private newtype TIREscapeAnalysisConfiguration = MkIREscapeAnalysisConfiguration
  * The query can extend this class to control what escape analysis is used when generating SSA.
  */
 class IREscapeAnalysisConfiguration extends TIREscapeAnalysisConfiguration {
+  /** Gets a textual representation of this element. */
   string toString() { result = "IREscapeAnalysisConfiguration" }
 
   /**

cpp/ql/src/semmle/code/cpp/ir/implementation/IRType.qll

@@ -32,6 +32,7 @@ private newtype TIRType =
  * all pointer types map to the same instance of `IRAddressType`.
  */
 class IRType extends TIRType {
+  /** Gets a textual representation of this type. */
   string toString() { none() }
 
   /**

cpp/ql/src/semmle/code/cpp/ir/implementation/MemoryAccessKind.qll

@@ -1,3 +1,9 @@
+/**
+ * Provides classes that describe how a particular `Instruction` or its operands access memory.
+ */
+
+private import IRConfiguration
+
 private newtype TMemoryAccessKind =
   TIndirectMemoryAccess() or
   TBufferMemoryAccess() or
@@ -14,6 +20,7 @@ private newtype TMemoryAccessKind =
  * memory result.
  */
 class MemoryAccessKind extends TMemoryAccessKind {
+  /** Gets a textual representation of this access kind. */
   string toString() { none() }
 
   /**