Add support for DuckDB underscore numeric literals (#891)

DuckDB allows underscores in numeric literals, e.g. `1_000_000`. This PR
adds support for this.

- Add `numberRegex` option to `TokenizerOptions` to allow custom number
patterns
- Update `Tokenizer` to use custom number regex when provided
- Configure DuckDB formatter to support underscore separators in numbers
- Add test for underscore numeric literals in DuckDB

Let me know if you'd rather see this done differently! Thanks for the
great library.

Author: nene

src/languages/duckdb/duckdb.formatter.ts

@@ -155,6 +155,9 @@ export const duckdb: DialectOptions = {
     reservedFunctionNames: functions,
     nestedBlockComments: true,
     extraParens: ['[]', '{}'],
+    // Support underscore separators in numeric literals (e.g., 1_000_000)
+    numberRegex:
+      /(?:0x[0-9a-fA-F_]+|0b[01_]+|(?:-\s*)?(?:[0-9_]*\.[0-9_]+|[0-9_]+(?:\.[0-9_]*)?)(?:[eE][-+]?[0-9_]+(?:\.[0-9_]+)?)?)(?![\w\p{Alphabetic}])/uy,
     stringTypes: [
       '$$',
       "''-qq",

src/lexer/Tokenizer.ts

@@ -51,6 +51,7 @@ export default class Tokenizer {
       {
         type: TokenType.NUMBER,
         regex:
+          cfg.numberRegex ??
           /(?:0x[0-9a-fA-F]+|0b[01]+|(?:-\s*)?(?:[0-9]*\.[0-9]+|[0-9]+(?:\.[0-9]*)?)(?:[eE][-+]?[0-9]+(?:\.[0-9]+)?)?)(?![\w\p{Alphabetic}])/uy,
       },
       // RESERVED_PHRASE is matched before all other keyword tokens

src/lexer/TokenizerOptions.ts

@@ -100,6 +100,8 @@ export interface TokenizerOptions {
   propertyAccessOperators?: string[];
   // Enables PostgreSQL-specific OPERATOR(...) syntax
   operatorKeyword?: boolean;
+  // Custom regex pattern for number tokens (defaults to standard SQL number pattern)
+  numberRegex?: RegExp;
   // Allows custom modifications on the token array.
   // Called after the whole input string has been split into tokens.
   // The result of this will be the output of the tokenizer.

test/duckdb.test.ts

@@ -214,4 +214,15 @@ describe('DuckDBFormatter', () => {
         1 IS NOT NULL;
     `);
   });
+
+  it('supports underscore separators in numeric literals', () => {
+    expect(format('SELECT 1_000_000, 3.14_159, 0x1A_2B_3C, 0b1010_0001, 1.5e+1_0;')).toBe(dedent`
+      SELECT
+        1_000_000,
+        3.14_159,
+        0x1A_2B_3C,
+        0b1010_0001,
+        1.5e+1_0;
+    `);
+  });
 });