AST: numeric separators, BigInt (#5272)

* Revert to more complicated lexing of numbers, as the Number constructor can't handle BigInts or numbers with numeric separators

* Add debugging information to error message test (#5239)

One of the test cases in test/error_messages.coffee fails intermittently
in the Node.js ecosystem-testing tool CITGM. In an effort to help debug
what's going on when this occurs, this adds more information to the
AssertionError message in question.

* Fix #5103: Add support for BigInt literals (#5104)

* Fix #5103: Add support for BigInt literals

* Fix typos found in testing

* Support binary, octal and hex BigInt literals

* Make decimal BigInt test consistent other bases

* Correct test BigInt test names

* Add Node versions to CI

* Numeric literal separators (#5215)

* implement numeric literal separators

* add tests

* Revert changes to package-lock.json

* small regex adjustment

* split tests

* add comment

* Add Node versions to CI

* Fix #5103: Add support for BigInt literals (#5104)

* Fix #5103: Add support for BigInt literals

* Fix typos found in testing

* Support binary, octal and hex BigInt literals

* Make decimal BigInt test consistent other bases

* Correct test BigInt test names

* Add Node versions to CI

* Update output

* Fix style

* support bigint literal with separators

* un-disallow property access on number literal

* Update output

* Refactor numeric literal separator tests to be more like the rest of the tests

* Add test for numeric property with underscore

Co-authored-by: Geoffrey Booth <[email protected]>
Co-authored-by: Robert de Forest <[email protected]>

* Update test style and output

* numeric separator parsed value

* BigInt AST; parseNumber()

Co-authored-by: Geoffrey Booth <[email protected]>
Co-authored-by: Rich Trott <[email protected]>
Co-authored-by: Robert de Forest <[email protected]>
Co-authored-by: square <[email protected]>

Author: 5 people

.travis.yml

@@ -4,6 +4,8 @@ node_js:
   - 6
   - 8
   - 10
+  - 12
+  - node # Latest
 
 cache:
   directories:

Cakefile

@@ -472,6 +472,8 @@ runTests = (CoffeeScript) ->
   skipUnless 'var a = 2 ** 2; a **= 3', ['exponentiation.coffee']
   skipUnless 'var {...a} = {}', ['object_rest_spread.coffee']
   skipUnless '/foo.bar/s.test("foo\tbar")', ['regex_dotall.coffee']
+  skipUnless '1_2_3', ['numeric_literal_separators.coffee']
+  skipUnless '1n', ['numbers_bigint.coffee']
   files = fs.readdirSync('test').filter (filename) ->
     filename not in testFilesToSkip
 

appveyor.yml

@@ -3,7 +3,8 @@ environment:
     - nodejs_version: '6'
     - nodejs_version: '8'
     - nodejs_version: '10'
-    - nodejs_version: '' # Installs latest.
+    - nodejs_version: '12'
+    - nodejs_version: '' # Latest
 
 install:
   - ps: Install-Product node $env:nodejs_version

lib/coffeescript/helpers.js

@@ -268,6 +268,20 @@ exports.nameWhitespaceCharacter = (string) ->
     when '\t' then 'tab'
     else string
 
+exports.parseNumber = (string) ->
+  return NaN unless string?
+
+  base = switch string.charAt 1
+    when 'b' then 2
+    when 'o' then 8
+    when 'x' then 16
+    else null
+
+  if base?
+    parseInt string[2..].replace(/_/g, ''), base
+  else
+    parseFloat string.replace(/_/g, '')
+
 exports.isFunction = (obj) -> Object::toString.call(obj) is '[object Function]'
 exports.isNumber = isNumber = (obj) -> Object::toString.call(obj) is '[object Number]'
 exports.isString = isString = (obj) -> Object::toString.call(obj) is '[object String]'

lib/coffeescript/lexer.js

@@ -14,7 +14,7 @@
 # Import the helpers we need.
 {count, starts, compact, repeat, invertLiterate, merge,
 attachCommentsToNode, locationDataToString, throwSyntaxError
-replaceUnicodeCodePointEscapes, flatten} = require './helpers'
+replaceUnicodeCodePointEscapes, flatten, parseNumber} = require './helpers'
 
 # The Lexer Class
 # ---------------
@@ -272,13 +272,7 @@ exports.Lexer = class Lexer
       when /^0\d+/.test number
         @error "octal literal '#{number}' must be prefixed with '0o'", length: lexedLength
 
-    base = switch number.charAt 1
-      when 'b' then 2
-      when 'o' then 8
-      when 'x' then 16
-      else null
-
-    parsedValue = if base? then parseInt(number[2..], base) else parseFloat(number)
+    parsedValue = parseNumber number
     tokenData = {parsedValue}
 
     tag = if parsedValue is Infinity then 'INFINITY' else 'NUMBER'
@@ -1302,10 +1296,14 @@ JSX_ATTRIBUTE = /// ^
 ///
 
 NUMBER     = ///
-  ^ 0b[01]+    |              # binary
-  ^ 0o[0-7]+   |              # octal
-  ^ 0x[\da-f]+ |              # hex
-  ^ \d*\.?\d+ (?:e[+-]?\d+)?  # decimal
+  ^ 0b[01](?:_?[01])*n?                         | # binary
+  ^ 0o[0-7](?:_?[0-7])*n?                       | # octal
+  ^ 0x[\da-f](?:_?[\da-f])*n?                   | # hex
+  ^ \d+n                                        | # decimal bigint
+  ^ (?:\d(?:_?\d)*)?    \.?   (?:\d(?:_?\d)*)+    # decimal
+                    (?:e[+-]? (?:\d(?:_?\d)*)+ )?
+  # decimal without support for numeric literal separators for reference:
+  # \d*\.?\d+ (?:e[+-]?\d+)?
 ///i
 
 OPERATOR   = /// ^ (

lib/coffeescript/nodes.js

@@ -12,7 +12,7 @@ Error.stackTraceLimit = Infinity
 {compact, flatten, extend, merge, del, starts, ends, some,
 addDataToNode, attachCommentsToNode, locationDataToString,
 throwSyntaxError, replaceUnicodeCodePointEscapes,
-isFunction, isPlainObject, isNumber} = require './helpers'
+isFunction, isPlainObject, isNumber, parseNumber} = require './helpers'
 
 # Functions required by parser.
 exports.extend = extend
@@ -936,15 +936,30 @@ exports.NumberLiteral = class NumberLiteral extends Literal
         @parsedValue = @value
         @value = "#{@value}"
       else
-        @parsedValue = Number @value
+        @parsedValue = parseNumber @value
 
-  astType: -> 'NumericLiteral'
+  isBigInt: ->
+    /n$/.test @value
+
+  astType: ->
+    if @isBigInt()
+      'BigIntLiteral'
+    else
+      'NumericLiteral'
 
   astProperties: ->
     return
-      value: @parsedValue
+      value:
+        if @isBigInt()
+          @parsedValue.toString()
+        else
+          @parsedValue
       extra:
-        rawValue: @parsedValue
+        rawValue:
+          if @isBigInt()
+            @parsedValue.toString()
+          else
+            @parsedValue
         raw: @value
 
 exports.InfinityLiteral = class InfinityLiteral extends NumberLiteral
@@ -2296,9 +2311,9 @@ exports.Range = class Range extends Base
     [@fromC, @fromVar] = @cacheToCodeFragments @from.cache o, LEVEL_LIST, shouldCache
     [@toC, @toVar]     = @cacheToCodeFragments @to.cache o, LEVEL_LIST, shouldCache
     [@step, @stepVar]  = @cacheToCodeFragments step.cache o, LEVEL_LIST, shouldCache if step = del o, 'step'
-    @fromNum = if @from.isNumber() then Number @fromVar else null
-    @toNum   = if @to.isNumber()   then Number @toVar   else null
-    @stepNum = if step?.isNumber() then Number @stepVar else null
+    @fromNum = if @from.isNumber() then parseNumber @fromVar else null
+    @toNum   = if @to.isNumber()   then parseNumber @toVar   else null
+    @stepNum = if step?.isNumber() then parseNumber @stepVar else null
 
   # When compiled normally, the range returns the contents of the *for loop*
   # needed to iterate over the values in the range. Used by comprehensions.
@@ -5362,7 +5377,7 @@ exports.For = class For extends While
     kvarAssign  = if kvar isnt ivar then "#{kvar} = " else ""
     if @step and not @range
       [step, stepVar] = @cacheToCodeFragments @step.cache o, LEVEL_LIST, shouldCacheOrIsAssignable
-      stepNum   = Number stepVar if @step.isNumber()
+      stepNum   = parseNumber stepVar if @step.isNumber()
     name        = ivar if @pattern
     varPart     = ''
     guardPart   = ''

src/helpers.coffee

@@ -177,6 +177,41 @@ test "AST as expected for NumberLiteral node", ->
       rawValue: 225
       raw: '0xE1'
 
+  testExpression '10_000',
+    type: 'NumericLiteral'
+    value: 10000
+    extra:
+      rawValue: 10000
+      raw: '10_000'
+
+  testExpression '1_2.34_5e6_7',
+    type: 'NumericLiteral'
+    value: 12.345e67
+    extra:
+      rawValue: 12.345e67
+      raw: '1_2.34_5e6_7'
+
+  testExpression '0o7_7_7',
+    type: 'NumericLiteral'
+    value: 0o777
+    extra:
+      rawValue: 0o777
+      raw: '0o7_7_7'
+
+  testExpression '42n',
+    type: 'BigIntLiteral'
+    value: '42'
+    extra:
+      rawValue: '42'
+      raw: '42n'
+
+  testExpression '2e3_08',
+    type: 'NumericLiteral'
+    value: Infinity
+    extra:
+      rawValue: Infinity
+      raw: '2e3_08'
+
 test "AST as expected for InfinityLiteral node", ->
   testExpression 'Infinity',
     type: 'Identifier'