Numeric literal separators (#5215)

* implement numeric literal separators

* add tests

* Revert changes to package-lock.json

* small regex adjustment

* split tests

* add comment

* Add Node versions to CI

* Fix #5103: Add support for BigInt literals (#5104)

* Fix #5103: Add support for BigInt literals

* Fix typos found in testing

* Support binary, octal and hex BigInt literals

* Make decimal BigInt test consistent other bases

* Correct test BigInt test names

* Add Node versions to CI

* Update output

* Fix style

* support bigint literal with separators

* un-disallow property access on number literal

* Update output

* Refactor numeric literal separator tests to be more like the rest of the tests

* Add test for numeric property with underscore

Co-authored-by: Geoffrey Booth <[email protected]>
Co-authored-by: Robert de Forest <[email protected]>

Author: 3 people

Cakefile

@@ -472,6 +472,7 @@ runTests = (CoffeeScript) ->
   skipUnless 'var a = 2 ** 2; a **= 3', ['exponentiation.coffee']
   skipUnless 'var {...a} = {}', ['object_rest_spread.coffee']
   skipUnless '/foo.bar/s.test("foo\tbar")', ['regex_dotall.coffee']
+  skipUnless '1_2_3', ['numeric_literal_separators.coffee']
   skipUnless '1n', ['numbers_bigint.coffee']
   files = fs.readdirSync('test').filter (filename) ->
     filename not in testFilesToSkip

lib/coffeescript/lexer.js

@@ -1212,11 +1212,14 @@ CSX_ATTRIBUTE = /// ^
 ///
 
 NUMBER     = ///
-  ^ 0b[01]+n?    |            # binary
-  ^ 0o[0-7]+n?   |            # octal
-  ^ 0x[\da-f]+n? |            # hex
-  ^ \d+n         |            # decimal bigint
-  ^ \d*\.?\d+ (?:e[+-]?\d+)?  # decimal
+  ^ 0b[01](?:_?[01])*n?                         | # binary
+  ^ 0o[0-7](?:_?[0-7])*n?                       | # octal
+  ^ 0x[\da-f](?:_?[\da-f])*n?                   | # hex
+  ^ \d+n                                        | # decimal bigint
+  ^ (?:\d(?:_?\d)*)?    \.?   (?:\d(?:_?\d)*)+    # decimal
+                    (?:e[+-]? (?:\d(?:_?\d)*)+ )?
+  # decimal without support for numeric literal separators for reference:
+  # \d*\.?\d+ (?:e[+-]?\d+)?
 ///i
 
 OPERATOR   = /// ^ (

src/lexer.coffee

@@ -0,0 +1,55 @@
+# Numeric Literal Separators
+# --------------------------
+
+test 'integer literals with separators', ->
+  eq 123_456, 123456
+  eq 12_34_56, 123456
+
+test 'decimal literals with separators', ->
+  eq 1_2.34_5, 12.345
+  eq 1_0e1_0, 10e10
+  eq 1_2.34_5e6_7, 12.345e67
+
+test 'hexadecimal literals with separators', ->
+  eq 0x1_2_3_4, 0x1234
+
+test 'binary literals with separators', ->
+  eq 0b10_10, 0b1010
+
+test 'octal literals with separators', ->
+  eq 0o7_7_7, 0o777
+
+test 'property access on a number', ->
+  # Somehow, `3..toFixed()` is valid JavaScript; though just `3.toFixed()`
+  # is not. CoffeeScript has long allowed code like `3.toFixed()` to compile
+  # into `3..toFixed()`.
+  eq 3.toFixed(), '3'
+  # Where this can conflict with numeric literal separators is when the
+  # property name contains an underscore.
+  Number::_23 = _23 = 'x'
+  eq 1._23, 'x'
+  ok 1._34 is undefined
+  delete Number::_23
+
+test 'invalid decimal literal separators do not compile', ->
+  # `1._23` is a valid property access (see previous test)
+  throws -> CoffeeScript.compile '1_.23'
+  throws -> CoffeeScript.compile '1e_2'
+  throws -> CoffeeScript.compile '1e2_'
+  throws -> CoffeeScript.compile '1_'
+  throws -> CoffeeScript.compile '1__2'
+
+test 'invalid hexadecimal literal separators do not compile', ->
+  throws -> CoffeeScript.compile '0x_1234'
+  throws -> CoffeeScript.compile '0x1234_'
+  throws -> CoffeeScript.compile '0x1__34'
+
+test 'invalid binary literal separators do not compile', ->
+  throws -> CoffeeScript.compile '0b_100'
+  throws -> CoffeeScript.compile '0b100_'
+  throws -> CoffeeScript.compile '0b1__1'
+
+test 'invalid octal literal separators do not compile', ->
+  throws -> CoffeeScript.compile '0o_777'
+  throws -> CoffeeScript.compile '0o777_'
+  throws -> CoffeeScript.compile '0o6__6'