Implement TabError detection

Author: msimacek

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/antlr/DescriptiveBailErrorListener.java

@@ -96,6 +96,10 @@ public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol,
                     entireMessage = "unindent does not match any outer indentation level";
                     errorType = ErrorType.Indentation;
                     break;
+                case Python3Parser.TAB_ERROR:
+                    entireMessage = "inconsistent use of tabs and spaces in indentation";
+                    errorType = ErrorType.Tab;
+                    break;
                 case Python3Parser.INDENT:
                     entireMessage = "unexpected indent";
                     errorType = ErrorType.Indentation;

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/antlr/Python3.g4

@@ -35,15 +35,26 @@ grammar Python3;
 // All comments that start with "///" are copy-pasted from
 // The Python Language Reference
 
-tokens { INDENT, DEDENT, INDENT_ERROR }
+tokens { INDENT, DEDENT, INDENT_ERROR, TAB_ERROR }
 
 @lexer::members {
   // new version with semantic actions in parser
 
+  private static class Indent {
+    public final int indent;
+    public final int altindent;
+    public static final Indent EMPTY = new Indent(0, 0);
+
+    public Indent(int indent, int altindent) {
+      this.indent = indent;
+      this.altindent = altindent;
+    }
+  }
+
   // A queue where extra tokens are pushed on (see the NEWLINE lexer rule).
   private java.util.LinkedList<Token> tokens = new java.util.LinkedList<>();
   // The stack that keeps track of the indentation level.
-  private java.util.Stack<Integer> indents = new java.util.Stack<>();
+  private java.util.Stack<Indent> indents = new java.util.Stack<>();
   // The amount of opened braces, brackets and parenthesis.
   private int opened = 0;
   // The most recently produced token.
@@ -167,7 +178,7 @@ tokens { INDENT, DEDENT, INDENT_ERROR }
     return dedent;
   }
 
-  private Token createIndentError() {
+  private Token createIndentError(int type) {
     // For some reason, CPython sets the error position to the end of line
     int cur = getCharIndex();
     String s;
@@ -176,7 +187,7 @@ tokens { INDENT, DEDENT, INDENT_ERROR }
         cur++;
     } while (!s.isEmpty() && s.charAt(0) != '\n');
     cur--;
-    CommonToken error = new CommonToken(this._tokenFactorySourcePair, Python3Parser.INDENT_ERROR, DEFAULT_TOKEN_CHANNEL, cur, cur);
+    CommonToken error = new CommonToken(this._tokenFactorySourcePair, type, DEFAULT_TOKEN_CHANNEL, cur, cur);
     error.setLine(this.lastToken.getLine());
     return error;
   }
@@ -194,9 +205,14 @@ tokens { INDENT, DEDENT, INDENT_ERROR }
   //  such that the total number of characters up to and including
   //  the replacement is a multiple of eight [...]"
   //
+  // Altindent is an alternative measure of spaces where tabs are
+  // counted as one space. The purpose is to validate that the code
+  // doesn't mix tabs and spaces in inconsistent way.
+  //
   //  -- https://docs.python.org/3.1/reference/lexical_analysis.html#indentation
-  static int getIndentationCount(String spaces) {
-    int count = 0;
+  static Indent getIndentationCount(String spaces) {
+    int indent = 0;
+    int altindent = 0;
     for (char ch : spaces.toCharArray()) {
       switch (ch) {
         case '\r':
@@ -205,15 +221,17 @@ tokens { INDENT, DEDENT, INDENT_ERROR }
           // ignore
           break;
         case '\t':
-          count += 8 - (count % 8);
+          indent += 8 - (indent % 8);
+          altindent++;
           break;
         default:
           // A normal space char.
-          count++;
+          indent++;
+          altindent++;
       }
     }
 
-    return count;
+    return new Indent(indent, altindent);
   }
 
   boolean atStartOfInput() {
@@ -1769,31 +1787,40 @@ NEWLINE
      }
      else {
        emit(commonToken(NEWLINE, "\n"));
-       int indent;
+       Indent indent;
        if (next == EOF) {
          // don't add indents if we're going to finish
-         indent = 0;
+         indent = Indent.EMPTY;
        } else {
          indent = getIndentationCount(getText());
        }
-       int previous = indents.isEmpty() ? 0 : indents.peek();
-       if (indent == previous) {
+       Indent previous = indents.isEmpty() ? Indent.EMPTY : indents.peek();
+       if (indent.indent == previous.indent) {
+         if (indent.altindent != previous.altindent) {
+           this.emit(createIndentError(Python3Parser.TAB_ERROR));
+         }
          // skip indents of the same size as the present indent-size
          skip();
        }
-       else if (indent > previous) {
+       else if (indent.indent > previous.indent) {
+         if (indent.altindent <= previous.altindent) {
+           this.emit(createIndentError(Python3Parser.TAB_ERROR));
+         }
          indents.push(indent);
          emit(commonToken(Python3Parser.INDENT, getText()));
        }
        else {
          // Possibly emit more than 1 DEDENT token.
-         while (!indents.isEmpty() && indents.peek() > indent) {
+         while (!indents.isEmpty() && indents.peek().indent > indent.indent) {
            this.emit(createDedent());
            indents.pop();
          }
-         int expectedIndent = indents.empty() ? 0 : indents.peek();
-         if (expectedIndent != indent) {
-           this.emit(createIndentError());
+         Indent expectedIndent = indents.isEmpty() ? Indent.EMPTY : indents.peek();
+         if (expectedIndent.indent != indent.indent) {
+           this.emit(createIndentError(Python3Parser.INDENT_ERROR));
+         }
+         if (expectedIndent.altindent != indent.altindent) {
+           this.emit(createIndentError(Python3Parser.TAB_ERROR));
          }
        }
      }