remove semicolon backward compatibility

Author: rafaelrpinto

src/core/Formatter.js

@@ -71,18 +71,18 @@ export default class Formatter {
             else if (token.type === tokenTypes.PLACEHOLDER) {
                 formattedQuery = this.formatPlaceholder(token, formattedQuery);
             }
-            else if (token.type === tokenTypes.END_STATEMENT) {
-                formattedQuery = this.formatWithoutSpaces(token, formattedQuery);
-            }
             else if (token.value === ",") {
                 formattedQuery = this.formatComma(token, formattedQuery);
             }
             else if (token.value === ":") {
                 formattedQuery = this.formatWithSpaceAfter(token, formattedQuery);
             }
-            else if (token.value === "." || token.value === ";") {
+            else if (token.value === ".") {
                 formattedQuery = this.formatWithoutSpaces(token, formattedQuery);
             }
+            else if (token.value === ";") {
+                formattedQuery = this.formatQuerySeparator(token, formattedQuery);
+            }
             else {
                 formattedQuery = this.formatWithSpaces(token, formattedQuery);
             }
@@ -188,6 +188,10 @@ export default class Formatter {
         return query + token.value + " ";
     }
 
+    formatQuerySeparator(token, query) {
+        return trimEnd(query) + token.value + "\n";
+    }
+
     addNewline(query) {
         return trimEnd(query) + "\n" + this.indentation.getIndent();
     }

src/core/Tokenizer.js

@@ -18,7 +18,6 @@ export default class Tokenizer {
      */
     constructor(cfg) {
         this.WHITESPACE_REGEX = /^(\s+)/;
-        this.END_STATEMENT_REGEX = /^(;\n)/;
         this.NUMBER_REGEX = /^((-\s*)?[0-9]+(\.[0-9]+)?|0x[0-9a-fA-F]+|0b[01]+)\b/;
         this.OPERATOR_REGEX = /^(!=|<>|==|<=|>=|!<|!>|\|\||::|->>|->|~~\*|~~|!~~\*|!~~|~\*|!~\*|!~|.)/;
 
@@ -134,7 +133,6 @@ export default class Tokenizer {
 
     getNextToken(input, previousToken) {
         return this.getWhitespaceToken(input) ||
-            this.getEndStatementToken(input) ||
             this.getCommentToken(input) ||
             this.getStringToken(input) ||
             this.getOpenParenToken(input) ||
@@ -154,14 +152,6 @@ export default class Tokenizer {
         });
     }
 
-    getEndStatementToken(input) {
-        return this.getTokenOnFirstMatch({
-            input,
-            type: tokenTypes.END_STATEMENT,
-            regex: this.END_STATEMENT_REGEX
-        });
-    }
-
     getCommentToken(input) {
         return this.getLineCommentToken(input) || this.getBlockCommentToken(input);
     }

src/core/tokenTypes.js

@@ -14,6 +14,5 @@ export default {
     LINE_COMMENT: "line-comment",
     BLOCK_COMMENT: "block-comment",
     NUMBER: "number",
-    PLACEHOLDER: "placeholder",
-    END_STATEMENT: "end-statement"
+    PLACEHOLDER: "placeholder"
 };

test/StandardSqlFormatterTest.js

@@ -358,8 +358,7 @@ describe("StandardSqlFormatter", function() {
             "SELECT\n" +
             "  a\n" +
             "FROM\n" +
-            "  b --comment\n" +
-            ";"
+            "  b --comment;"
         );
     });
 

test/behavesLikeSqlFormatter.js

@@ -447,10 +447,9 @@ export default function behavesLikeSqlFormatter(language) {
     });
 
     it("keeps separation between multiple statements", function() {
-        expect(format("foo;bar;")).toBe("foo;bar;");
-        expect(format("foo\n;bar;")).toBe("foo;bar;");
-        expect(format("foo;\nbar;\n\n\n")).toBe("foo;\nbar;");
-        expect(format("foo;\n\n\nbar;\n\n\n")).toBe("foo;\nbar;");
+        expect(format("foo;bar;")).toBe("foo;\nbar;");
+        expect(format("foo\n;bar;")).toBe("foo;\nbar;");
+        expect(format("foo\n\n\n;bar;\n\n")).toBe("foo;\nbar;");
 
         const result = format("SELECT count(*),Column1 FROM Table1;\nSELECT count(*),Column1 FROM Table2;");
         expect(result).toBe(