Ability to log comments

Signed-off-by: worksofliam <[email protected]>

Author: worksofliam

src/language/sql/document.ts

@@ -6,11 +6,13 @@ export default class Document {
   content: string;
   statements: Statement[];
 
-  constructor(content: string) {
+  constructor(content: string, keepComments = false) {
     this.content = content;
     this.statements = [];
 
     const tokeniser = new SQLTokeniser();
+    tokeniser.storeComments = keepComments;
+
     this.parseStatements(tokeniser.tokenise(content));
   }
 

src/language/sql/tests/tokens.test.ts

@@ -27,6 +27,19 @@ test('Comment test', () => {
   expect(tokens.length).toBe(10);
 });
 
+test('Comment token test', () => {
+  const tokeniser = new SQLTokeniser();
+  tokeniser.storeComments = true;
+
+  const tokens = tokeniser.tokenise([
+    `--hello: world!!!: coolness`,
+    `select * from table(func()) x`
+  ].join(`\n`));
+
+  expect(tokens.length).toBe(12);
+  expect(tokens.some(t => t.value === `--hello: world!!!: coolness`));
+});
+
 test('New line (\\n) and comments test', () => {
   const tokeniser = new SQLTokeniser();
 

src/language/sql/tokens.ts

@@ -25,7 +25,7 @@ interface TokenState {
 }
 
 export default class SQLTokeniser {
-  matchers: Matcher[] = [
+  static matchers: Matcher[] = [
     {
       name: `PROCEDURE_PARM_TYPE`,
       match: [{ type: `word`, match: (value: string) => {return [`IN`, `OUT`, `INOUT`].includes(value.toUpperCase())}}],
@@ -145,6 +145,8 @@ export default class SQLTokeniser {
   readonly startCommentBlock = `/*`;
   readonly endCommentBlock = `*/`;
 
+  storeComments: boolean = false;
+
   constructor() { }
 
   tokenise(content: string) {
@@ -166,6 +168,11 @@ export default class SQLTokeniser {
         // Handle when the end of line is there and we're in a comment
       } else if (state === ReadState.IN_SIMPLE_COMMENT && content[i] === this.endCommentString) {
         const preNewLine = i - 1;
+
+        if (this.storeComments) {
+          result.push({ value: content.substring(commentStart, i), type: `comment`, range: { start: commentStart, end: i } });
+        }
+
         content = content.substring(0, commentStart) + ` `.repeat(preNewLine - commentStart) + content.substring(preNewLine);
         i--; // So we process the newline next
         state = ReadState.NORMAL;
@@ -268,8 +275,8 @@ export default class SQLTokeniser {
     let tokens = state.tokens;
 
     for (let i = 0; i < tokens.length; i++) {
-      for (let y = 0; y < this.matchers.length; y++) {
-        const type = this.matchers[y];
+      for (let y = 0; y < SQLTokeniser.matchers.length; y++) {
+        const type = SQLTokeniser.matchers[y];
         let goodMatch = true;
 
         for (let x = 0; x < type.match.length; x++) {