feat: print fixed tokens regardless of span

stuff like punctuation doesn't need to have a valid span to know what
should be printed. this makes it cheaper to build arbitrary cst's
without needing to allocate string space for known grammar symbols

Author: alanpq

crates/ltk_ritobin/src/parse/tokenizer.rs

@@ -27,6 +27,35 @@ impl TokenKind {
     pub fn is_string(&self) -> bool {
         matches!(self, Self::String | Self::UnterminatedString)
     }
+
+    pub fn print_value(&self) -> Option<&'static str> {
+        match self {
+            TokenKind::Unknown => None,
+            TokenKind::Eof => None,
+            TokenKind::Newline => None,
+            TokenKind::LParen => Some("("),
+            TokenKind::RParen => Some(")"),
+            TokenKind::LCurly => Some("{"),
+            TokenKind::RCurly => Some("}"),
+            TokenKind::LBrack => Some("["),
+            TokenKind::RBrack => Some("]"),
+            TokenKind::Eq => Some("="),
+            TokenKind::Comma => Some(","),
+            TokenKind::Colon => Some(":"),
+            TokenKind::SemiColon => Some(";"),
+            TokenKind::Star => Some("*"),
+            TokenKind::Slash => Some("/"),
+            TokenKind::Quote => Some("\""),
+            TokenKind::String => None,
+            TokenKind::UnterminatedString => None,
+            TokenKind::Comment => None,
+            TokenKind::True => Some("true"),
+            TokenKind::False => Some("false"),
+            TokenKind::Name => None,
+            TokenKind::Number => None,
+            TokenKind::HexLit => None,
+        }
+    }
 }
 
 impl Display for TokenKind {

crates/ltk_ritobin/src/print/visitor.rs

@@ -378,8 +378,9 @@ impl<'a, W: Write> Printer<'a, W> {
         context: &Cst,
     ) -> Result<(), PrintError> {
         let txt = self.src[token.span].trim();
+        let print_value = token.kind.print_value();
 
-        if txt.is_empty() {
+        if txt.is_empty() && print_value.is_none() {
             return Ok(());
         }
 
@@ -414,6 +415,22 @@ impl<'a, W: Write> Printer<'a, W> {
                 self.space();
             }
 
+            TokenKind::LBrack => {
+                self.text("[");
+            }
+            TokenKind::RBrack => {
+                self.text("]");
+            }
+            TokenKind::Quote => {
+                self.text("\"");
+            }
+            TokenKind::False => {
+                self.text("false");
+            }
+            TokenKind::True => {
+                self.text("true");
+            }
+
             _ => {
                 self.text(txt);
             }