QL: Integrate blame parser into extractor

Author: tausbn

ql/autobuilder/src/main.rs

@@ -19,6 +19,7 @@ fn main() -> std::io::Result<()> {
         .arg("--include-extension=.qll")
         .arg("--include-extension=.dbscheme")
         .arg("--include=**/qlpack.yml")
+        .arg("--include=deprecated.blame")
         .arg("--size-limit=5m")
         .arg("--language=ql")
         .arg("--working-dir=.")

ql/extractor/Cargo.toml

@@ -13,6 +13,7 @@ tree-sitter = ">= 0.20, < 0.21"
 tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql.git", rev = "d08db734f8dc52f6bc04db53a966603122bc6985"}
 tree-sitter-ql-dbscheme = { git = "https://github.com/erik-krogh/tree-sitter-ql-dbscheme.git", rev = "63e1344353f63931e88bfbc2faa2e78e1421b213"}
 tree-sitter-ql-yaml = {git = "https://github.com/erik-krogh/tree-sitter-ql.git", rev = "cf704bf3671e1ae148e173464fb65a4d2bbf5f99"}
+tree-sitter-blame = {path = "../buramu/tree-sitter-blame"}
 clap = "2.33"
 tracing = "0.1"
 tracing-subscriber = { version = "0.3.16", features = ["env-filter"] }

ql/extractor/src/main.rs

@@ -87,10 +87,12 @@ fn main() -> std::io::Result<()> {
     let language = tree_sitter_ql::language();
     let dbscheme = tree_sitter_ql_dbscheme::language();
     let yaml = tree_sitter_ql_yaml::language();
+    let blame = tree_sitter_blame::language();
     let schema = node_types::read_node_types_str("ql", tree_sitter_ql::NODE_TYPES)?;
     let dbscheme_schema =
         node_types::read_node_types_str("dbscheme", tree_sitter_ql_dbscheme::NODE_TYPES)?;
     let yaml_schema = node_types::read_node_types_str("yaml", tree_sitter_ql_yaml::NODE_TYPES)?;
+    let blame_schema = node_types::read_node_types_str("blame", tree_sitter_blame::NODE_TYPES)?;
 
     let lines: std::io::Result<Vec<String>> = std::io::BufReader::new(file_list).lines().collect();
     let lines = lines?;
@@ -103,6 +105,7 @@ fn main() -> std::io::Result<()> {
                 && !line.ends_with(".qll")
                 && !line.ends_with(".dbscheme")
                 && !line.ends_with("qlpack.yml")
+                && !line.ends_with(".blame")
             {
                 return Ok(());
             }
@@ -131,6 +134,16 @@ fn main() -> std::io::Result<()> {
                     &source,
                     &code_ranges,
                 )?
+            } else if line.ends_with(".blame") {
+                extractor::extract(
+                    blame,
+                    "blame",
+                    &blame_schema,
+                    &mut trap_writer,
+                    &path,
+                    &source,
+                    &code_ranges,
+                )?
             } else {
                 extractor::extract(
                     language,

ql/generator/Cargo.toml

@@ -13,4 +13,5 @@ tracing = "0.1"
 tracing-subscriber = { version = "0.3.16", features = ["env-filter"] }
 tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql.git", rev = "d08db734f8dc52f6bc04db53a966603122bc6985"}
 tree-sitter-ql-dbscheme = { git = "https://github.com/erik-krogh/tree-sitter-ql-dbscheme.git", rev = "63e1344353f63931e88bfbc2faa2e78e1421b213"}
-tree-sitter-ql-yaml = {git = "https://github.com/erik-krogh/tree-sitter-ql.git", rev = "cf704bf3671e1ae148e173464fb65a4d2bbf5f99"}
+tree-sitter-ql-yaml = {git = "https://github.com/erik-krogh/tree-sitter-ql.git", rev = "cf704bf3671e1ae148e173464fb65a4d2bbf5f99"}
+tree-sitter-blame = {path = "../buramu/tree-sitter-blame"}

ql/generator/src/main.rs

@@ -577,6 +577,10 @@ fn main() -> std::io::Result<()> {
             name: "Yaml".to_owned(),
             node_types: tree_sitter_ql_yaml::NODE_TYPES,
         },
+        Language {
+            name: "Blame".to_owned(),
+            node_types: tree_sitter_blame::NODE_TYPES,
+        },
     ];
     let mut dbscheme_writer = LineWriter::new(File::create(dbscheme_path)?);
     write!(

ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll

@@ -1743,3 +1743,116 @@ module Yaml {
     final override AstNode getAFieldOrChild() { yaml_yaml_child(this, _, result) }
   }
 }
+
+module Blame {
+  /** The base class for all AST nodes */
+  class AstNode extends @blame_ast_node {
+    /** Gets a string representation of this element. */
+    string toString() { result = this.getAPrimaryQlClass() }
+
+    /** Gets the location of this element. */
+    final L::Location getLocation() { blame_ast_node_info(this, _, _, result) }
+
+    /** Gets the parent of this element. */
+    final AstNode getParent() { blame_ast_node_info(this, result, _, _) }
+
+    /** Gets the index of this node among the children of its parent. */
+    final int getParentIndex() { blame_ast_node_info(this, _, result, _) }
+
+    /** Gets a field or child node of this node. */
+    AstNode getAFieldOrChild() { none() }
+
+    /** Gets the name of the primary QL class for this element. */
+    string getAPrimaryQlClass() { result = "???" }
+
+    /** Gets a comma-separated list of the names of the primary CodeQL classes to which this element belongs. */
+    string getPrimaryQlClasses() { result = concat(this.getAPrimaryQlClass(), ",") }
+  }
+
+  /** A token. */
+  class Token extends @blame_token, AstNode {
+    /** Gets the value of this token. */
+    final string getValue() { blame_tokeninfo(this, _, result) }
+
+    /** Gets a string representation of this element. */
+    final override string toString() { result = this.getValue() }
+
+    /** Gets the name of the primary QL class for this element. */
+    override string getAPrimaryQlClass() { result = "Token" }
+  }
+
+  /** A reserved word. */
+  class ReservedWord extends @blame_reserved_word, Token {
+    /** Gets the name of the primary QL class for this element. */
+    final override string getAPrimaryQlClass() { result = "ReservedWord" }
+  }
+
+  /** A class representing `blame_entry` nodes. */
+  class BlameEntry extends @blame_blame_entry, AstNode {
+    /** Gets the name of the primary QL class for this element. */
+    final override string getAPrimaryQlClass() { result = "BlameEntry" }
+
+    /** Gets the node corresponding to the field `date`. */
+    final Date getDate() { blame_blame_entry_def(this, result) }
+
+    /** Gets the node corresponding to the field `line`. */
+    final Number getLine(int i) { blame_blame_entry_line(this, i, result) }
+
+    /** Gets a field or child node of this node. */
+    final override AstNode getAFieldOrChild() {
+      blame_blame_entry_def(this, result) or blame_blame_entry_line(this, _, result)
+    }
+  }
+
+  /** A class representing `blame_info` nodes. */
+  class BlameInfo extends @blame_blame_info, AstNode {
+    /** Gets the name of the primary QL class for this element. */
+    final override string getAPrimaryQlClass() { result = "BlameInfo" }
+
+    /** Gets the node corresponding to the field `file_entry`. */
+    final FileEntry getFileEntry(int i) { blame_blame_info_file_entry(this, i, result) }
+
+    /** Gets the node corresponding to the field `today`. */
+    final Date getToday() { blame_blame_info_def(this, result) }
+
+    /** Gets a field or child node of this node. */
+    final override AstNode getAFieldOrChild() {
+      blame_blame_info_file_entry(this, _, result) or blame_blame_info_def(this, result)
+    }
+  }
+
+  /** A class representing `date` tokens. */
+  class Date extends @blame_token_date, Token {
+    /** Gets the name of the primary QL class for this element. */
+    final override string getAPrimaryQlClass() { result = "Date" }
+  }
+
+  /** A class representing `file_entry` nodes. */
+  class FileEntry extends @blame_file_entry, AstNode {
+    /** Gets the name of the primary QL class for this element. */
+    final override string getAPrimaryQlClass() { result = "FileEntry" }
+
+    /** Gets the node corresponding to the field `blame_entry`. */
+    final BlameEntry getBlameEntry(int i) { blame_file_entry_blame_entry(this, i, result) }
+
+    /** Gets the node corresponding to the field `file_name`. */
+    final Filename getFileName() { blame_file_entry_def(this, result) }
+
+    /** Gets a field or child node of this node. */
+    final override AstNode getAFieldOrChild() {
+      blame_file_entry_blame_entry(this, _, result) or blame_file_entry_def(this, result)
+    }
+  }
+
+  /** A class representing `filename` tokens. */
+  class Filename extends @blame_token_filename, Token {
+    /** Gets the name of the primary QL class for this element. */
+    final override string getAPrimaryQlClass() { result = "Filename" }
+  }
+
+  /** A class representing `number` tokens. */
+  class Number extends @blame_token_number, Token {
+    /** Gets the name of the primary QL class for this element. */
+    final override string getAPrimaryQlClass() { result = "Number" }
+  }
+}

ql/ql/src/ql.dbscheme

@@ -1187,3 +1187,65 @@ yaml_ast_node_info(
   int loc: @location ref
 );
 
+#keyset[blame_blame_entry, index]
+blame_blame_entry_line(
+  int blame_blame_entry: @blame_blame_entry ref,
+  int index: int ref,
+  unique int line: @blame_token_number ref
+);
+
+blame_blame_entry_def(
+  unique int id: @blame_blame_entry,
+  int date__: @blame_token_date ref
+);
+
+#keyset[blame_blame_info, index]
+blame_blame_info_file_entry(
+  int blame_blame_info: @blame_blame_info ref,
+  int index: int ref,
+  unique int file_entry: @blame_file_entry ref
+);
+
+blame_blame_info_def(
+  unique int id: @blame_blame_info,
+  int today: @blame_token_date ref
+);
+
+#keyset[blame_file_entry, index]
+blame_file_entry_blame_entry(
+  int blame_file_entry: @blame_file_entry ref,
+  int index: int ref,
+  unique int blame_entry: @blame_blame_entry ref
+);
+
+blame_file_entry_def(
+  unique int id: @blame_file_entry,
+  int file_name: @blame_token_filename ref
+);
+
+blame_tokeninfo(
+  unique int id: @blame_token,
+  int kind: int ref,
+  string value: string ref
+);
+
+case @blame_token.kind of
+  0 = @blame_reserved_word
+| 1 = @blame_token_date
+| 2 = @blame_token_filename
+| 3 = @blame_token_number
+;
+
+
+@blame_ast_node = @blame_blame_entry | @blame_blame_info | @blame_file_entry | @blame_token
+
+@blame_ast_node_parent = @blame_ast_node | @file
+
+#keyset[parent, parent_index]
+blame_ast_node_info(
+  unique int node: @blame_ast_node ref,
+  int parent: @blame_ast_node_parent ref,
+  int parent_index: int ref,
+  int loc: @location ref
+);
+