fix(tokenizer): fix token col attribute when there is leading whitespace after a newline (#5094)

Author: chgiff

sqlglot/tokens.py

@@ -1109,7 +1109,7 @@ def _advance(self, i: int = 1, alnum: bool = False) -> None:
         if self.WHITE_SPACE.get(self._char) is TokenType.BREAK:
             # Ensures we don't count an extra line if we get a \r\n line break sequence
             if not (self._char == "\r" and self._peek == "\n"):
-                self._col = 1
+                self._col = i
                 self._line += 1
         else:
             self._col += i

sqlglotrs/src/tokenizer.rs

@@ -173,7 +173,7 @@ impl<'a> TokenizerState<'a> {
         if Some(&self.token_types.break_) == self.settings.white_space.get(&self.current_char) {
             // Ensures we don't count an extra line if we get a \r\n line break sequence.
             if !(self.current_char == '\r' && self.peek_char == '\n') {
-                self.column = 1;
+                self.column = i as usize;
                 self.line += 1;
             }
         } else {

tests/test_tokens.py

@@ -67,9 +67,19 @@ def test_token_line_col(self):
         tokens = Tokenizer().tokenize("SELECT\r\n  1,\r\n  2")
 
         self.assertEqual(tokens[0].line, 1)
+        self.assertEqual(tokens[0].col, 6)
         self.assertEqual(tokens[1].line, 2)
+        self.assertEqual(tokens[1].col, 3)
         self.assertEqual(tokens[2].line, 2)
+        self.assertEqual(tokens[2].col, 4)
         self.assertEqual(tokens[3].line, 3)
+        self.assertEqual(tokens[3].col, 3)
+
+        tokens = Tokenizer().tokenize("  SELECT\n    100")
+        self.assertEqual(tokens[0].line, 1)
+        self.assertEqual(tokens[0].col, 8)
+        self.assertEqual(tokens[1].line, 2)
+        self.assertEqual(tokens[1].col, 7)
 
     def test_crlf(self):
         tokens = Tokenizer().tokenize("SELECT a\r\nFROM b")