Feat(sqlglotrs): match the Python implementation of __repr__ for tokens (#5172)

Author: georgesittas

sqlglotrs/src/token.rs

@@ -1,7 +1,7 @@
 use crate::settings::TokenType;
 use pyo3::prelude::*;
 use pyo3::types::{PyList, PyString};
-use pyo3::{pyclass, Py, PyObject, Python};
+use pyo3::{pyclass, pymethods, Py, PyObject, Python};
 
 #[derive(Debug)]
 #[pyclass]
@@ -57,3 +57,25 @@ impl Token {
         });
     }
 }
+
+#[pymethods]
+impl Token {
+    fn __repr__(&self, py: Python) -> PyResult<String> {
+        let text = self.text.bind(py).to_str()?;
+        let comments = self.comments.bind(py);
+        let token_type_str = self.token_type_py.bind(py).str()?;
+        let comments_repr = comments.repr()?;
+        let comments_str = comments_repr.to_str()?;
+
+        Ok(format!(
+            "<Token token_type: {}, text: {}, line: {}, col: {}, start: {}, end: {}, comments: {}>",
+            token_type_str,
+            text,
+            self.line,
+            self.col,
+            self.start,
+            self.end,
+            comments_str
+        ))
+    }
+}

tests/test_tokens.py

@@ -201,3 +201,10 @@ def test_partial_token_list(self):
         self.assertEqual(len(partial_tokens), 1)
         self.assertEqual(partial_tokens[0].token_type, TokenType.VAR)
         self.assertEqual(partial_tokens[0].text, "foo")
+
+    def test_token_repr(self):
+        # Ensures both the Python and the Rust tokenizer produce a human-friendly representation
+        self.assertEqual(
+            repr(Tokenizer().tokenize("foo")),
+            "[<Token token_type: TokenType.VAR, text: foo, line: 1, col: 3, start: 0, end: 2, comments: []>]",
+        )