[_fe_analyzer_shared] Shared literal parsing

This adds shared helpers for parsing bool, int, and double literals to
their corresponding values.

The macro metadata literals are updated support digit separators and
to hold the literal value.

Change-Id: I709018b6c3a3b8ab09188d36c28f3c09636e0ea0
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/391964
Reviewed-by: Konstantin Shcheglov <[email protected]>
Reviewed-by: Jens Johansen <[email protected]>
Commit-Queue: Johnni Winther <[email protected]>

Author: johnniwinther

pkg/_fe_analyzer_shared/lib/src/metadata/expressions.dart

@@ -90,21 +90,25 @@ class ConstructorInvocation extends Expression {
 }
 
 class IntegerLiteral extends Expression {
-  final String text;
+  final String? text;
+  final int? value;
+
+  IntegerLiteral.fromText(String this.text, [this.value]);
 
-  IntegerLiteral(this.text);
+  IntegerLiteral.fromValue(int this.value) : text = null;
 
   @override
-  String toString() => 'IntegerLiteral($text)';
+  String toString() => 'IntegerLiteral(${value ?? text})';
 
   @override
   Expression? resolve() => null;
 }
 
 class DoubleLiteral extends Expression {
   final String text;
+  final double value;
 
-  DoubleLiteral(this.text);
+  DoubleLiteral(this.text, this.value);
 
   @override
   String toString() => 'DoubleLiteral($text)';
@@ -114,12 +118,12 @@ class DoubleLiteral extends Expression {
 }
 
 class BooleanLiteral extends Expression {
-  final String text;
+  final bool value;
 
-  BooleanLiteral(this.text);
+  BooleanLiteral(this.value);
 
   @override
-  String toString() => 'BooleanLiteral($text)';
+  String toString() => 'BooleanLiteral($value)';
 
   @override
   Expression? resolve() => null;

pkg/_fe_analyzer_shared/lib/src/metadata/parser.dart

@@ -577,17 +577,31 @@ class AnnotationsListener extends StackListener {
 
   @override
   void handleLiteralInt(Token token) {
-    push(new ExpressionProto(new IntegerLiteral(token.lexeme)));
+    int? value = intFromToken(token, hasSeparators: false);
+    push(new ExpressionProto(new IntegerLiteral.fromText(token.lexeme, value)));
+  }
+
+  @override
+  void handleLiteralIntWithSeparators(Token token) {
+    int? value = intFromToken(token, hasSeparators: true);
+    push(new ExpressionProto(new IntegerLiteral.fromText(token.lexeme, value)));
   }
 
   @override
   void handleLiteralDouble(Token token) {
-    push(new ExpressionProto(new DoubleLiteral(token.lexeme)));
+    push(new ExpressionProto(new DoubleLiteral(
+        token.lexeme, doubleFromToken(token, hasSeparators: false))));
+  }
+
+  @override
+  void handleLiteralDoubleWithSeparators(Token token) {
+    push(new ExpressionProto(new DoubleLiteral(
+        token.lexeme, doubleFromToken(token, hasSeparators: true))));
   }
 
   @override
   void handleLiteralBool(Token token) {
-    push(new ExpressionProto(new BooleanLiteral(token.lexeme)));
+    push(new ExpressionProto(new BooleanLiteral(boolFromToken(token))));
   }
 
   @override
@@ -953,7 +967,8 @@ enum _NullValues implements NullValue<Object> {
   TypeAnnotations,
 }
 
-final Argument _dummyArgument = new PositionalArgument(new IntegerLiteral('0'));
+final Argument _dummyArgument =
+    new PositionalArgument(new IntegerLiteral.fromText('0'));
 
 final RecordField _dummyRecordField =
     new RecordPositionalField(_dummyExpression);

pkg/_fe_analyzer_shared/lib/src/parser/parser.dart

@@ -41,7 +41,15 @@ export 'parser_error.dart' show ParserError;
 
 export 'top_level_parser.dart' show TopLevelParser;
 
-export 'util.dart' show lengthForToken, lengthOfSpan, optional;
+export 'util.dart'
+    show
+        boolFromToken,
+        doubleFromToken,
+        intFromToken,
+        lengthForToken,
+        lengthOfSpan,
+        optional,
+        stripSeparators;
 
 class ErrorCollectingListener extends Listener {
   final List<ParserError> recoverableErrors = <ParserError>[];

pkg/_fe_analyzer_shared/lib/src/parser/util.dart

@@ -8,7 +8,7 @@ import 'dart:typed_data';
 
 import '../messages/codes.dart' show noLength;
 
-import '../scanner/scanner.dart' show Token;
+import '../scanner/scanner.dart' show Keyword, Token;
 
 import '../scanner/token.dart'
     show BeginToken, SimpleToken, SyntheticToken, TokenIsAExtension, TokenType;
@@ -248,3 +248,39 @@ Token syntheticGt(Token next) {
     // so that the previous token is not set.
     ..next = next;
 }
+
+/// Returns the boolean value from a 'true' or 'false' [token].
+bool boolFromToken(Token token) {
+  bool value = token.isA(Keyword.TRUE);
+  assert(value || token.isA(Keyword.FALSE));
+  return value;
+}
+
+/// Returns the integer value from an integer literal token.
+///
+/// If [hasSeparators], separator characters, '_', are stripped before parsing
+/// the token text.
+///
+/// `null` is returned if the token text could not be parsed as an integer
+/// value. This does _not_ mean that the token is not valid as an integer token
+/// since negated integer literals are parsed as a unary operation on the
+/// positive integer.
+int? intFromToken(Token token, {required bool hasSeparators}) {
+  String text = token.lexeme;
+  if (hasSeparators) {
+    text = stripSeparators(text);
+  }
+  return int.tryParse(text);
+}
+
+/// Returns the double value from an double literal token.
+///
+/// If [hasSeparators], separator characters, '_', are stripped before parsing
+/// the token text.
+double doubleFromToken(Token token, {required bool hasSeparators}) {
+  String text = token.lexeme;
+  if (hasSeparators) {
+    text = stripSeparators(text);
+  }
+  return double.parse(text);
+}

pkg/_fe_analyzer_shared/lib/src/testing/metadata_helper.dart

@@ -124,15 +124,19 @@ class Writer {
         _write(')');
       case IntegerLiteral():
         _write('IntegerLiteral(');
-        _write(expression.text);
+        if (expression.text != null) {
+          _write(expression.text!);
+        } else {
+          _write('value=${expression.value}');
+        }
         _write(')');
       case DoubleLiteral():
         _write('DoubleLiteral(');
         _write(expression.text);
         _write(')');
       case BooleanLiteral():
         _write('BooleanLiteral(');
-        _write(expression.text);
+        _write('${expression.value}');
         _write(')');
       case NullLiteral():
         _write('NullLiteral()');

pkg/analyzer/lib/src/fasta/ast_builder.dart

@@ -35,18 +35,20 @@ import 'package:_fe_analyzer_shared/src/parser/parser.dart'
     show
         Assert,
         BlockKind,
+        boolFromToken,
         ConstructorReferenceContext,
         DeclarationHeaderKind,
         DeclarationKind,
+        doubleFromToken,
         FormalParameterKind,
         IdentifierContext,
+        intFromToken,
         MemberKind,
         optional,
         Parser;
 import 'package:_fe_analyzer_shared/src/parser/quote.dart';
 import 'package:_fe_analyzer_shared/src/parser/stack_listener.dart'
     show NullValues, StackListener;
-import 'package:_fe_analyzer_shared/src/parser/util.dart' show stripSeparators;
 import 'package:_fe_analyzer_shared/src/scanner/errors.dart'
     show translateErrorToken;
 import 'package:_fe_analyzer_shared/src/scanner/scanner.dart';
@@ -4708,8 +4710,7 @@ class AstBuilder extends StackListener {
 
   @override
   void handleLiteralBool(Token token) {
-    bool value = identical(token.stringValue, "true");
-    assert(value || identical(token.stringValue, "false"));
+    bool value = boolFromToken(token);
     debugEvent("LiteralBool");
 
     push(
@@ -4745,14 +4746,10 @@ class AstBuilder extends StackListener {
       );
     }
 
-    // Only copy `source` if we find a separator ('_'). Most int literals
-    // will not have any separator, and so a quick scan will show we do not
-    // need to produce a new String.
-    var source = stripSeparators(token.lexeme);
     push(
       DoubleLiteralImpl(
         literal: token,
-        value: double.parse(source),
+        value: doubleFromToken(token, hasSeparators: true),
       ),
     );
   }
@@ -4766,7 +4763,7 @@ class AstBuilder extends StackListener {
     push(
       IntegerLiteralImpl(
         literal: token,
-        value: int.tryParse(token.lexeme),
+        value: intFromToken(token, hasSeparators: false),
       ),
     );
   }
@@ -4784,11 +4781,10 @@ class AstBuilder extends StackListener {
       );
     }
 
-    var source = stripSeparators(token.lexeme);
     push(
       IntegerLiteralImpl(
         literal: token,
-        value: int.tryParse(source),
+        value: intFromToken(token, hasSeparators: true),
       ),
     );
   }

pkg/front_end/lib/src/kernel/body_builder.dart

@@ -14,8 +14,12 @@ import 'package:_fe_analyzer_shared/src/parser/parser.dart'
         IdentifierContext,
         MemberKind,
         Parser,
+        boolFromToken,
+        doubleFromToken,
+        intFromToken,
         lengthForToken,
-        lengthOfSpan;
+        lengthOfSpan,
+        stripSeparators;
 import 'package:_fe_analyzer_shared/src/parser/quote.dart'
     show
         Quote,
@@ -26,7 +30,6 @@ import 'package:_fe_analyzer_shared/src/parser/quote.dart'
         unescapeString;
 import 'package:_fe_analyzer_shared/src/parser/stack_listener.dart'
     show FixedNullableList, GrowableList, NullValues, ParserRecovery;
-import 'package:_fe_analyzer_shared/src/parser/util.dart' show stripSeparators;
 import 'package:_fe_analyzer_shared/src/scanner/token.dart'
     show Keyword, Token, TokenIsAExtension, TokenType;
 import 'package:_fe_analyzer_shared/src/scanner/token_impl.dart'
@@ -3526,7 +3529,7 @@ class BodyBuilder extends StackListenerImpl
   @override
   void handleLiteralInt(Token token) {
     debugEvent("LiteralInt");
-    int? value = int.tryParse(token.lexeme);
+    int? value = intFromToken(token, hasSeparators: false);
     // Postpone parsing of literals resulting in a negative value
     // (hex literals >= 2^63). These are only allowed when not negated.
     if (value == null || value < 0) {
@@ -4795,16 +4798,15 @@ class BodyBuilder extends StackListenerImpl
   @override
   void handleLiteralBool(Token token) {
     debugEvent("LiteralBool");
-    bool value = token.isA(Keyword.TRUE);
-    assert(value || token.isA(Keyword.FALSE));
+    bool value = boolFromToken(token);
     push(forest.createBoolLiteral(offsetForToken(token), value));
   }
 
   @override
   void handleLiteralDouble(Token token) {
     debugEvent("LiteralDouble");
     push(forest.createDoubleLiteral(
-        offsetForToken(token), double.parse(token.lexeme)));
+        offsetForToken(token), doubleFromToken(token, hasSeparators: false)));
   }
 
   @override
@@ -4819,8 +4821,7 @@ class BodyBuilder extends StackListenerImpl
           token.length);
     }
 
-    String source = stripSeparators(token.lexeme);
-    double value = double.parse(source);
+    double value = doubleFromToken(token, hasSeparators: true);
     push(forest.createDoubleLiteral(offsetForToken(token), value));
   }
 

pkg/front_end/lib/src/kernel/macro/annotation_parser.dart

@@ -6,7 +6,6 @@ import 'package:_fe_analyzer_shared/src/experiments/flags.dart';
 import 'package:_fe_analyzer_shared/src/messages/codes.dart';
 import 'package:_fe_analyzer_shared/src/parser/parser.dart';
 import 'package:_fe_analyzer_shared/src/parser/quote.dart';
-import 'package:_fe_analyzer_shared/src/parser/util.dart' show stripSeparators;
 import 'package:_fe_analyzer_shared/src/scanner/error_token.dart';
 import 'package:_fe_analyzer_shared/src/scanner/token.dart';
 import 'package:macros/src/executor.dart' as macro;
@@ -406,33 +405,31 @@ class _MacroListener implements Listener {
 
   @override
   void handleLiteralBool(Token token) {
-    push(
-        new _MacroArgumentNode(new macro.BoolArgument(token.lexeme == 'true')));
+    push(new _MacroArgumentNode(new macro.BoolArgument(boolFromToken(token))));
   }
 
   @override
   void handleLiteralDouble(Token token) {
-    push(new _MacroArgumentNode(
-        new macro.DoubleArgument(double.parse(token.lexeme))));
+    push(new _MacroArgumentNode(new macro.DoubleArgument(
+        doubleFromToken(token, hasSeparators: false))));
   }
 
   @override
   void handleLiteralDoubleWithSeparators(Token token) {
-    String source = stripSeparators(token.lexeme);
-    push(
-        new _MacroArgumentNode(new macro.DoubleArgument(double.parse(source))));
+    push(new _MacroArgumentNode(new macro.DoubleArgument(
+        doubleFromToken(token, hasSeparators: true))));
   }
 
   @override
   void handleLiteralInt(Token token) {
-    push(
-        new _MacroArgumentNode(new macro.IntArgument(int.parse(token.lexeme))));
+    push(new _MacroArgumentNode(new macro.IntArgument(
+        intFromToken(token, hasSeparators: false)!)));
   }
 
   @override
   void handleLiteralIntWithSeparators(Token token) {
-    String source = stripSeparators(token.lexeme);
-    push(new _MacroArgumentNode(new macro.IntArgument(int.parse(source))));
+    push(new _MacroArgumentNode(
+        new macro.IntArgument(intFromToken(token, hasSeparators: true)!)));
   }
 
   @override