Correctly format array type specifier (#91)

Author: lu-zero

.github/workflows/sqlformat.yml

@@ -20,7 +20,7 @@ jobs:
           - latest-nightly
         include:
           - conf: minimum
-            toolchain: 1.65.0
+            toolchain: 1.84.0
           - conf: latest-stable
             toolchain: stable
           - conf: latest-beta
@@ -50,16 +50,7 @@ jobs:
         uses: actions-rs-plus/clippy-check@v2.3.0
         with:
           args: --all -- -D warnings
-      # FIXME: criterion and its dependencies require a newer version than 1.65, but it is only used for benchmarks.
-      # Is there a way to not have criterion built when we run tests?
-      - name: Run cargo check
-        if: matrix.toolchain == '1.65.0'
-        run: cargo check
       - name: Run tests
-        if: matrix.toolchain != '1.65.0'
         run: cargo test
-      - name: Build benchmarks
-        if: matrix.toolchain == 'stable'
-        run: cargo bench --no-run
       - name: Build docs
         run: cargo doc --no-deps

Cargo.toml

@@ -3,7 +3,7 @@ name = "sqlformat"
 version = "0.3.5"
 authors = ["Josh Holmer <jholmer.in@gmail.com>"]
 edition = "2021"
-rust-version = "1.65"
+rust-version = "1.84"
 license = "MIT OR Apache-2.0"
 homepage = "https://github.com/shssoichiro/sqlformat-rs"
 repository = "https://github.com/shssoichiro/sqlformat-rs"

src/formatter.rs

@@ -76,7 +76,6 @@ pub(crate) fn format(
             formatter.format_no_change(token, &mut formatted_query);
             continue;
         }
-
         match token.kind {
             TokenKind::Whitespace => {
                 // ignore (we do our own whitespace formatting)
@@ -112,8 +111,8 @@ pub(crate) fn format(
             TokenKind::Placeholder => {
                 formatter.format_placeholder(token, &mut formatted_query);
             }
-            TokenKind::DoubleColon => {
-                formatter.format_double_colon(token, &mut formatted_query);
+            TokenKind::TypeSpecifier => {
+                formatter.format_type_specifier(token, &mut formatted_query);
             }
             _ => match token.value {
                 "," => {
@@ -166,11 +165,11 @@ impl<'a> Formatter<'a> {
 
     fn format_line_comment(&mut self, token: &Token<'_>, query: &mut String) {
         let is_whitespace_followed_by_special_token =
-            self.next_token(1).map_or(false, |current_token| {
+            self.next_token(1).is_some_and(|current_token| {
                 current_token.kind == TokenKind::Whitespace
-                    && self.next_token(2).map_or(false, |next_token| {
-                        !matches!(next_token.kind, TokenKind::Operator)
-                    })
+                    && self
+                        .next_token(2)
+                        .is_some_and(|next_token| !matches!(next_token.kind, TokenKind::Operator))
             });
 
         let previous_token = self.previous_token(1);
@@ -189,9 +188,9 @@ impl<'a> Formatter<'a> {
         self.add_new_line(query);
     }
 
-    fn format_double_colon(&self, _token: &Token<'_>, query: &mut String) {
+    fn format_type_specifier(&self, token: &Token<'_>, query: &mut String) {
         self.trim_all_spaces_end(query);
-        query.push_str("::");
+        query.push_str(token.value);
     }
     fn format_block_comment(&mut self, token: &Token<'_>, query: &mut String) {
         self.add_new_line(query);
@@ -211,7 +210,7 @@ impl<'a> Formatter<'a> {
                 true,
                 self.options
                     .max_inline_top_level
-                    .map_or(true, |limit| limit < span_len),
+                    .is_none_or(|limit| limit < span_len),
             )
         }
     }
@@ -254,7 +253,7 @@ impl<'a> Formatter<'a> {
             && self
                 .options
                 .max_inline_arguments
-                .map_or(true, |limit| limit < self.indentation.span())
+                .is_none_or(|limit| limit < self.indentation.span())
         {
             self.add_new_line(query);
         } else {

src/lib.rs

@@ -397,6 +397,22 @@ mod tests {
         assert_eq!(format(input, &QueryParams::None, &options), expected);
     }
 
+    #[test]
+    fn it_formats_type_specifiers() {
+        let input = "SELECT id,  ARRAY [] :: UUID [] FROM UNNEST($1  ::  UUID   []);";
+        let options = FormatOptions::default();
+        let expected = indoc!(
+            "
+            SELECT
+              id,
+              ARRAY[]::UUID[]
+            FROM
+              UNNEST($1::UUID[]);"
+        );
+
+        assert_eq!(format(input, &QueryParams::None, &options), expected);
+    }
+
     #[test]
     fn it_formats_limit_of_single_value_and_offset() {
         let input = "LIMIT 5 OFFSET 8;";

src/params.rs

@@ -12,7 +12,7 @@ impl<'a> Params<'a> {
     }
 
     pub fn get(&mut self, token: &'a Token<'a>) -> &'a str {
-        let named_placeholder_token = token.key.as_ref().map_or(false, |key| key.named() != "");
+        let named_placeholder_token = token.key.as_ref().is_some_and(|key| key.named() != "");
 
         match self.params {
             QueryParams::Named(params) => token

src/tokenizer.rs

@@ -17,6 +17,7 @@ pub(crate) fn tokenize<'a>(
 ) -> Vec<Token<'a>> {
     let mut tokens: Vec<Token> = Vec::new();
 
+    let mut last_non_whitespace_token = None;
     let mut last_reserved_token = None;
     let mut last_reserved_top_level_token = None;
 
@@ -27,7 +28,7 @@ pub(crate) fn tokenize<'a>(
     // Keep processing the string until it is empty
     while let Ok(mut result) = get_next_token(
         &mut input,
-        tokens.last().cloned(),
+        last_non_whitespace_token.clone(),
         last_reserved_token.clone(),
         last_reserved_top_level_token.clone(),
         named_placeholders,
@@ -49,6 +50,10 @@ pub(crate) fn tokenize<'a>(
             _ => {}
         }
 
+        if result.kind != TokenKind::Whitespace {
+            last_non_whitespace_token = Some(result.clone());
+        }
+
         tokens.push(result);
 
         if let Ok(Some(result)) = opt(get_whitespace_token).parse_next(&mut input) {
@@ -68,7 +73,7 @@ pub(crate) struct Token<'a> {
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub(crate) enum TokenKind {
-    DoubleColon,
+    TypeSpecifier,
     Whitespace,
     String,
     Reserved,
@@ -119,6 +124,7 @@ fn get_next_token<'a>(
 ) -> Result<Token<'a>> {
     alt((
         get_comment_token,
+        |input: &mut _| get_type_specifier_token(input, previous_token.clone()),
         get_string_token,
         get_open_paren_token,
         get_close_paren_token,
@@ -131,20 +137,36 @@ fn get_next_token<'a>(
                 last_reserved_top_level_token.clone(),
             )
         },
-        get_double_colon_token,
         get_operator_token,
         |input: &mut _| get_placeholder_token(input, named_placeholders),
         get_word_token,
         get_any_other_char,
     ))
     .parse_next(input)
 }
-fn get_double_colon_token<'i>(input: &mut &'i str) -> Result<Token<'i>> {
-    "::".parse_next(input).map(|token| Token {
-        kind: TokenKind::DoubleColon,
-        value: token,
-        key: None,
-    })
+fn get_type_specifier_token<'i>(
+    input: &mut &'i str,
+    previous_token: Option<Token<'i>>,
+) -> Result<Token<'i>> {
+    if previous_token.is_some_and(|token| {
+        ![
+            TokenKind::CloseParen,
+            TokenKind::Placeholder,
+            TokenKind::Reserved,
+            TokenKind::String,
+            TokenKind::TypeSpecifier,
+            TokenKind::Word,
+        ]
+        .contains(&token.kind)
+    }) {
+        fail.parse_next(input)
+    } else {
+        alt(("::", "[]")).parse_next(input).map(|token| Token {
+            kind: TokenKind::TypeSpecifier,
+            value: token,
+            key: None,
+        })
+    }
 }
 fn get_whitespace_token<'i>(input: &mut &'i str) -> Result<Token<'i>> {
     take_while(1.., char::is_whitespace)