Fix(tokenizer)!: conditionally consume +/- in scientific literal notation (#6610)

Author: georgesittas

sqlglot/tokens.py

@@ -1363,8 +1363,12 @@ def _scan_number(self) -> None:
                 decimal = True
                 self._advance()
             elif self._peek in ("-", "+") and scientific == 1:
-                scientific += 1
-                self._advance()
+                # Only consume +/- if followed by a digit
+                if self._current + 1 < self.size and self.sql[self._current + 1].isdigit():
+                    scientific += 1
+                    self._advance()
+                else:
+                    return self._add(TokenType.NUMBER)
             elif self._peek.upper() == "E" and not scientific:
                 scientific += 1
                 self._advance()

sqlglotrs/src/tokenizer.rs

@@ -513,8 +513,13 @@ impl<'a> TokenizerState<'a> {
                 decimal = true;
                 self.advance(1)?;
             } else if (self.peek_char == '-' || self.peek_char == '+') && scientific == 1 {
-                scientific += 1;
-                self.advance(1)?;
+                // Only consume +/- if followed by a digit
+                if self.current + 1 < self.size && self.sql[self.current + 1].is_ascii_digit() {
+                    scientific += 1;
+                    self.advance(1)?;
+                } else {
+                    return self.add(self.token_types.number, None);
+                }
             } else if self.peek_char.to_ascii_uppercase() == 'E' && scientific == 0 {
                 scientific += 1;
                 self.advance(1)?;

tests/dialects/test_duckdb.py

@@ -369,6 +369,10 @@ def test_duckdb(self):
             parse_one("a // b", read="duckdb").assert_is(exp.IntDiv).sql(dialect="duckdb"), "a // b"
         )
 
+        self.validate_identity(
+            "SELECT tbl.x*1e4+tbl.y FROM tbl",
+            "SELECT tbl.x * 1e4 + tbl.y FROM tbl",
+        )
         self.validate_identity("DAYNAME(x)")
         self.validate_identity("MONTHNAME(x)")
         self.validate_identity(