Implement Numeric Separator

Add support for numeric separator characters in numeric literals. These are just syntactic sugar which improve readability of numeric constants in source. They desugar completely out of the numbers when parsed.

Numeric digits in the constant may have a single '_' character between them. The constant may not begin or end with a '_' character and there may not be multiple numeric separators in a row. All numeric constants are supported. Includes decimal, hex, octal and binary numeric constants.

```javascript
1234 === 1_2_3_4; // true
12.34e56 === 1_2.3_4e5_6; // true
0xff === 0xf_f; // true
0o17 === 0o1_7; // true
0b11 === 0b1_1; // true
```

Numeric Separator proposal is in stage 3 and implemented by JSC and V8. See proposal:
https://github.com/tc39/proposal-numeric-separator

Fixes: #6060

Author: boingoing

bin/NativeTests/BigUIntTest.h

@@ -26,7 +26,7 @@ namespace Js
 }
 
 template <typename EncodedChar>
-double Js::NumberUtilities::StrToDbl(const EncodedChar *, const EncodedChar **, LikelyNumberType& , bool)
+double Js::NumberUtilities::StrToDbl(const EncodedChar *, const EncodedChar **, LikelyNumberType& , bool, bool)
 {
     Assert(false);
     return 0.0;// don't care

lib/Common/Common/NumberUtilities.h

@@ -227,7 +227,7 @@ namespace Js
 
         // Implemented in lib\parser\common.  Should move to lib\common
         template<typename EncodedChar>
-        static double StrToDbl(const EncodedChar *psz, const EncodedChar **ppchLim, LikelyNumberType& likelyType, bool isESBigIntEnabled = false);
+        static double StrToDbl(const EncodedChar *psz, const EncodedChar **ppchLim, LikelyNumberType& likelyType, bool isESBigIntEnabled = false, bool isNumericSeparatorEnabled = false);
 
         static BOOL FDblToStr(double dbl, __out_ecount(nDstBufSize) char16 *psz, int nDstBufSize);
         static int FDblToStr(double dbl, NumberUtilities::FormatType ft, int nDigits, __out_ecount(cchDst) char16 *pchDst, int cchDst);

lib/Common/Common/NumberUtilities_strtod.cpp

@@ -366,7 +366,7 @@ void BIGNUM::SetFromRgchExp(const EncodedChar *prgch, int32 cch, int32 lwExp)
 
     while (++prgch < pchLim)
     {
-        if (*prgch == '.')
+        if (*prgch == '.' || *prgch == '_')
             continue;
         Assert(Js::NumberUtilities::IsDigit(*prgch));
         MulTenAdd((byte) (*prgch - '0'), &luExtra);
@@ -893,7 +893,7 @@ static double AdjustDbl(double dbl, const EncodedChar *prgch, int32 cch, int32 l
 String to Double.
 ***************************************************************************/
 template <typename EncodedChar>
-double Js::NumberUtilities::StrToDbl( const EncodedChar *psz, const EncodedChar **ppchLim, LikelyNumberType& likelyNumberType, bool isBigIntEnabled)
+double Js::NumberUtilities::StrToDbl(const EncodedChar *psz, const EncodedChar **ppchLim, LikelyNumberType& likelyNumberType, bool isBigIntEnabled, bool isNumericSeparatorEnabled)
 {
     uint32 lu;
     BIGNUM num;
@@ -909,6 +909,10 @@ double Js::NumberUtilities::StrToDbl( const EncodedChar *psz, const EncodedChar
     Assert(Js::NumberUtilities::IsNan(dblLowPrec));
 #endif //DBG
 
+    // Numeric separator characters exist in the numeric constant and should
+    // be ignored.
+    bool hasNumericSeparators = false;
+
     // For the mantissa digits. After leaving the state machine, pchMinDig
     // points to the first digit and pchLimDig points just past the last
     // digit. cchDig is the number of digits. pchLimDig - pchMinDig may be
@@ -979,8 +983,11 @@ double Js::NumberUtilities::StrToDbl( const EncodedChar *psz, const EncodedChar
     if (Js::NumberUtilities::IsDigit(*pch))
     {
 LGetLeftDig:
-        pchMinDig = pch;
-        for (cchDig = 1; Js::NumberUtilities::IsDigit(*++pch); cchDig++)
+        if (pchMinDig == NULL)
+        {
+            pchMinDig = pch;
+        }
+        for (cchDig++; Js::NumberUtilities::IsDigit(*++pch); cchDig++)
             ;
     }
     switch (*pch)
@@ -995,6 +1002,26 @@ double Js::NumberUtilities::StrToDbl( const EncodedChar *psz, const EncodedChar
         {
             goto LBigInt;
         }
+        goto LGetLeftDefault;
+    case '_':
+        if (isNumericSeparatorEnabled)
+        {
+            // A numeric separator is only valid if it appears between two
+            // digits. If the preceeding or following character is not a digit,
+            // we should just fallthrough and fail. Otherwise we would have to
+            // handle cases like 1_.0 manually above.
+            // cchDig holds the count of digits in the literal. If it's >0, we
+            // can be sure the previous pch is valid.
+            if (cchDig > 0 && Js::NumberUtilities::IsDigit(*(pch - 1))
+                && Js::NumberUtilities::IsDigit(*(pch + 1)))
+            {
+                hasNumericSeparators = true;
+                pch++;
+                goto LGetLeftDig;
+            }
+        }
+        // Fallthrough
+LGetLeftDefault:
     default:
         likelyNumberType = LikelyNumberType::Int;
     }
@@ -1010,6 +1037,7 @@ double Js::NumberUtilities::StrToDbl( const EncodedChar *psz, const EncodedChar
             lwAdj--;
         pchMinDig = pch;
     }
+LGetRightDigit:
     for( ; Js::NumberUtilities::IsDigit(*pch); pch++)
     {
         cchDig++;
@@ -1020,6 +1048,17 @@ double Js::NumberUtilities::StrToDbl( const EncodedChar *psz, const EncodedChar
     case 'E':
     case 'e':
         goto LGetExp;
+    case '_':
+        if (isNumericSeparatorEnabled)
+        {
+            if (cchDig > 0 && Js::NumberUtilities::IsDigit(*(pch - 1)) &&
+                Js::NumberUtilities::IsDigit(*(pch + 1)))
+            {
+                hasNumericSeparators = true;
+                pch++;
+                goto LGetRightDigit;
+            }
+        }
     }
     goto LEnd;
 
@@ -1050,6 +1089,19 @@ double Js::NumberUtilities::StrToDbl( const EncodedChar *psz, const EncodedChar
         if (lwExp > 100000000)
             lwExp = 100000000;
     }
+    switch (*pch)
+    {
+    case '_':
+        if (isNumericSeparatorEnabled)
+        {
+            if (Js::NumberUtilities::IsDigit(*(pch - 1)) &&
+                Js::NumberUtilities::IsDigit(*(pch + 1)))
+            {
+                pch++;
+                goto LGetExpDigits;
+            }
+        }
+    }
     goto LEnd;
 
 LBigInt:
@@ -1070,7 +1122,8 @@ double Js::NumberUtilities::StrToDbl( const EncodedChar *psz, const EncodedChar
         pchLimDig = pch;
     Assert(pchMinDig != NULL);
     Assert(pchLimDig - pchMinDig == cchDig ||
-        pchLimDig - pchMinDig == cchDig + 1);
+        pchLimDig - pchMinDig == cchDig + 1 ||
+        (isNumericSeparatorEnabled && hasNumericSeparators));
 
     // Limit to kcchMaxSig digits.
     if (cchDig > kcchMaxSig)
@@ -1116,15 +1169,16 @@ double Js::NumberUtilities::StrToDbl( const EncodedChar *psz, const EncodedChar
             cchDig--;
             lwAdj++;
         }
-        else if (*pchLimDig != '.')
+        else if (*pchLimDig != '.' && *pchLimDig != '_')
         {
             Assert(FNzDigit(*pchLimDig));
             pchLimDig++;
             break;
         }
     }
     Assert(pchLimDig - pchMinDig == cchDig ||
-        pchLimDig - pchMinDig == cchDig + 1);
+        pchLimDig - pchMinDig == cchDig + 1 ||
+        (isNumericSeparatorEnabled && hasNumericSeparators));
 
     if (signExp < 0)
         lwExp = -lwExp;
@@ -1139,8 +1193,14 @@ double Js::NumberUtilities::StrToDbl( const EncodedChar *psz, const EncodedChar
             // Can use the ALU.
             for (lu = 0, pch = pchMinDig; pch < pchLimDig; pch++)
             {
-                if (*pch != '.')
+                switch (*pch)
                 {
+                case '.':
+                    break;
+                case '_':
+                    Assert(isNumericSeparatorEnabled && hasNumericSeparators);
+                    break;
+                default:
                     Assert(Js::NumberUtilities::IsDigit(*pch));
                     lu = lu * 10 + (*pch - '0');
                 }
@@ -1151,8 +1211,14 @@ double Js::NumberUtilities::StrToDbl( const EncodedChar *psz, const EncodedChar
         {
             for (dbl = 0, pch = pchMinDig; pch < pchLimDig; pch++)
             {
-                if (*pch != '.')
+                switch (*pch)
                 {
+                case '.':
+                    break;
+                case '_':
+                    Assert(isNumericSeparatorEnabled && hasNumericSeparators);
+                    break;
+                default:
                     Assert(Js::NumberUtilities::IsDigit(*pch));
                     dbl = dbl * 10 + (*pch - '0');
                 }
@@ -1270,8 +1336,8 @@ double Js::NumberUtilities::StrToDbl( const EncodedChar *psz, const EncodedChar
     return dbl;
 }
 
-template double Js::NumberUtilities::StrToDbl<char16>( const char16 * psz, const char16 **ppchLim, LikelyNumberType& likelyInt, bool isBigIntEnabled );
-template double Js::NumberUtilities::StrToDbl<utf8char_t>(const utf8char_t * psz, const utf8char_t **ppchLim, LikelyNumberType& likelyInt, bool isBigIntEnabled );
+template double Js::NumberUtilities::StrToDbl<char16>( const char16 * psz, const char16 **ppchLim, LikelyNumberType& likelyInt, bool isBigIntEnabled, bool isNumericSeparatorEnabled );
+template double Js::NumberUtilities::StrToDbl<utf8char_t>(const utf8char_t * psz, const utf8char_t **ppchLim, LikelyNumberType& likelyInt, bool isBigIntEnabled, bool isNumericSeparatorEnabled );
 
 /***************************************************************************
 Uses big integer arithmetic to get the sequence of digits.

lib/Common/ConfigFlagsList.h

@@ -679,6 +679,7 @@ PHASE(All)
 #define DEFAULT_CONFIG_ES6RegExSticky          (true)
 #define DEFAULT_CONFIG_ES2018RegExDotAll       (true)
 #define DEFAULT_CONFIG_ESBigInt                (false)
+#define DEFAULT_CONFIG_ESNumericSeparator      (true)
 #define DEFAULT_CONFIG_ESSymbolDescription     (true)
 #define DEFAULT_CONFIG_ESGlobalThis            (true)
 #ifdef COMPILE_DISABLE_ES6RegExPrototypeProperties
@@ -1215,6 +1216,9 @@ FLAGR(Boolean, WinRTAdaptiveApps        , "Enable the adaptive apps feature, all
 // ES BigInt flag
 FLAGR(Boolean, ESBigInt, "Enable ESBigInt flag", DEFAULT_CONFIG_ESBigInt)
 
+// ES Numeric Separator support for numeric constants
+FLAGR(Boolean, ESNumericSeparator, "Enable Numeric Separator flag", DEFAULT_CONFIG_ESNumericSeparator)
+
 // ES Symbol.prototype.description flag
 FLAGR(Boolean, ESSymbolDescription, "Enable Symbol.prototype.description", DEFAULT_CONFIG_ESSymbolDescription)
 

lib/Common/DataStructures/BigUInt.cpp

@@ -125,7 +125,7 @@ namespace Js
         luMul = 1;
         for (*pcchDig = cch; prgch < pchLim; prgch++)
         {
-            if (*prgch == '.')
+            if (*prgch == '.' || *prgch == '_')
             {
                 (*pcchDig)--;
                 continue;

lib/Parser/Scan.cpp

@@ -679,7 +679,7 @@ typename Scanner<EncodingPolicy>::EncodedCharPtr Scanner<EncodingPolicy>::FScanN
     else
     {
 LFloat:
-        *pdbl = Js::NumberUtilities::StrToDbl(p, &pchT, likelyType, m_scriptContext->GetConfig()->IsESBigIntEnabled());
+        *pdbl = Js::NumberUtilities::StrToDbl(p, &pchT, likelyType, m_scriptContext->GetConfig()->IsESBigIntEnabled(), m_scriptContext->GetConfig()->IsESNumericSeparatorEnabled());
         Assert(pchT == p || !Js::NumberUtilities::IsNan(*pdbl));
         if (likelyType == LikelyNumberType::BigInt)
         {

lib/Runtime/Base/ThreadConfigFlagsList.h

@@ -48,6 +48,7 @@ FLAG_RELEASE(IsESObjectGetOwnPropertyDescriptorsEnabled, ESObjectGetOwnPropertyD
 FLAG_RELEASE(IsESSharedArrayBufferEnabled, ESSharedArrayBuffer)
 FLAG_RELEASE(IsESDynamicImportEnabled, ESDynamicImport)
 FLAG_RELEASE(IsESBigIntEnabled, ESBigInt)
+FLAG_RELEASE(IsESNumericSeparatorEnabled, ESNumericSeparator)
 FLAG_RELEASE(IsESExportNsAsEnabled, ESExportNsAs)
 FLAG_RELEASE(IsESSymbolDescriptionEnabled, ESSymbolDescription)
 FLAG_RELEASE(IsESGlobalThisEnabled, ESGlobalThis)

test/Number/NumericSeparator.js

@@ -0,0 +1,70 @@
+//-------------------------------------------------------------------------------------------------------
+// Copyright (C) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE.txt file in the project root for full license information.
+//-------------------------------------------------------------------------------------------------------
+
+WScript.LoadScriptFile("..\\UnitTestFramework\\UnitTestFramework.js");
+
+var tests = [
+  {
+    name: "Basic decimal support",
+    body: function () {
+        assert.areEqual(1234, 1_234, "1234 === 1_234");
+        assert.areEqual(1234, 1_2_3_4, "1234 === 1_2_3_4");
+        assert.areEqual(1234.567, 1_2_3_4.5_6_7, "1234.567 === 1_2_3_4.5_6_7");
+
+        assert.areEqual(-1234, -1_2_34, "-1234 === -1_2_34");
+        assert.areEqual(-12.34, -1_2.3_4, "-12.34 === -1_2.3_4");
+    }
+  },
+  {
+    name: "Decimal with exponent",
+    body: function () {
+        assert.areEqual(1e100, 1e1_00, "1e100 === 1e1_00");
+        assert.areEqual(Infinity, 1e1_0_0_0, "Infinity === 1e1_0_0_0");
+
+        assert.areEqual(123.456e23, 1_2_3.4_5_6e2_3, "123.456e23 === 1_2_3.4_5_6e2_3");
+        assert.areEqual(123.456e001, 1_2_3.4_5_6e0_0_1, "123.456e001 === 1_2_3.4_5_6e0_0_1");
+    }
+  },
+  {
+    name: "Decimal bad syntax",
+    body: function () {
+        // Decimal left-part only with numeric separators
+        assert.throws(()=>eval('1__2'), SyntaxError, "Multiple numeric separators in a row are now allowed");
+        assert.throws(()=>eval('1_2____3'), SyntaxError, "Multiple numeric separators in a row are now allowed");
+        assert.throws(()=>eval('1_'), SyntaxError, "Decimal may not end in a numeric separator");
+        assert.throws(()=>eval('1__'), SyntaxError, "Decimal may not end in a numeric separator");
+        assert.throws(()=>eval('__1'), ReferenceError, "Decimal may not begin with a numeric separator");
+        assert.throws(()=>eval('_1'), ReferenceError, "Decimal may not begin with a numeric separator");
+
+        // Decimal with right-part with numeric separators
+        assert.throws(()=>eval('1.0__0'), SyntaxError, "Decimal right-part may not contain multiple contiguous numeric separators");
+        assert.throws(()=>eval('1.0_0__2'), SyntaxError, "Decimal right-part may not contain multiple contiguous numeric separators");
+        assert.throws(()=>eval('1._'), SyntaxError, "Decimal right-part may not be a single numeric separator");
+        assert.throws(()=>eval('1.__'), SyntaxError, "Decimal right-part may not be multiple numeric separators");
+        assert.throws(()=>eval('1._0'), SyntaxError, "Decimal right-part may not begin with a numeric separator");
+        assert.throws(()=>eval('1.__0'), SyntaxError, "Decimal right-part may not begin with a numeric separator");
+        assert.throws(()=>eval('1.0_'), SyntaxError, "Decimal right-part may not end with a numeric separator");
+        assert.throws(()=>eval('1.0__'), SyntaxError, "Decimal right-part may not end with a numeric separator");
+
+        // Decimal with both parts with numeric separators
+        assert.throws(()=>eval('1_.0'), SyntaxError, "Decimal left-part may not end in numeric separator");
+        assert.throws(()=>eval('1__.0'), SyntaxError, "Decimal left-part may not end in numeric separator");
+        assert.throws(()=>eval('1__2.0'), SyntaxError, "Decimal left-part may not contain multiple contiguous numeric separators");
+
+        // Decimal with exponent with numeric separators
+        assert.throws(()=>eval('1_e10'), SyntaxError, "Decimal left-part may not end in numeric separator");
+        assert.throws(()=>eval('1e_1'), SyntaxError, "Exponent may not begin with numeric separator");
+        assert.throws(()=>eval('1e__1'), SyntaxError, "Exponent may not begin with numeric separator");
+        assert.throws(()=>eval('1e1_'), SyntaxError, "Exponent may not end with numeric separator");
+        assert.throws(()=>eval('1e1__'), SyntaxError, "Exponent may not end with numeric separator");
+        assert.throws(()=>eval('1e1__2'), SyntaxError, "Exponent may not contain multiple contiguous numeric separators");
+
+        // Decimal big ints with numeric separators
+        assert.throws(()=>eval('1_n'), SyntaxError);
+    }
+  },
+];
+
+testRunner.runTests(tests, { verbose: WScript.Arguments[0] != "summary" });

test/Number/rlexe.xml

@@ -63,4 +63,10 @@
       <compile-flags>-args summary -endargs</compile-flags>
     </default>
   </test>
+  <test>
+    <default>
+      <files>NumericSeparator.js</files>
+      <compile-flags>-ESNumericSeparator -args summary -endargs</compile-flags>
+    </default>
+  </test>
 </regress-exe>