Reapply "[LLDB] Add ScalarLiteralNode and literal parsing in DIL" (#155610)

This patch introduces `ScalarLiteralNode` without any uses by other
nodes yet. It also includes lexing and parsing for integer and floating
point numbers.
Reapplies #152308 with a fix.

Author: kuilpd

lldb/docs/dil-expr-lang.ebnf

@@ -15,7 +15,8 @@ postfix_expression = primary_expression
                    | postfix_expression "." id_expression
                    | postfix_expression "->" id_expression ;
 
-primary_expression = id_expression
+primary_expression = numeric_literal
+                   | id_expression
                    | "(" expression ")" ;
 
 id_expression = unqualified_id
@@ -31,6 +32,9 @@ identifier = ? C99 Identifier ? ;
 
 integer_literal = ? Integer constant: hexademical, decimal, octal, binary ? ;
 
+numeric_literal = ? Integer constant: hexademical, decimal, octal, binary ?
+                | ? Floating constant ? ;
+
 register = "$" ? Register name ? ;
 
 nested_name_specifier = type_name "::"

lldb/include/lldb/ValueObject/DILAST.h

@@ -21,7 +21,9 @@ enum class NodeKind {
   eArraySubscriptNode,
   eBitExtractionNode,
   eErrorNode,
+  eFloatLiteralNode,
   eIdentifierNode,
+  eIntegerLiteralNode,
   eMemberOfNode,
   eUnaryOpNode,
 };
@@ -178,6 +180,52 @@ class BitFieldExtractionNode : public ASTNode {
   int64_t m_last_index;
 };
 
+enum class IntegerTypeSuffix { None, Long, LongLong };
+
+class IntegerLiteralNode : public ASTNode {
+public:
+  IntegerLiteralNode(uint32_t location, llvm::APInt value, uint32_t radix,
+                     bool is_unsigned, IntegerTypeSuffix type)
+      : ASTNode(location, NodeKind::eIntegerLiteralNode),
+        m_value(std::move(value)), m_radix(radix), m_is_unsigned(is_unsigned),
+        m_type(type) {}
+
+  llvm::Expected<lldb::ValueObjectSP> Accept(Visitor *v) const override;
+
+  const llvm::APInt &GetValue() const { return m_value; }
+  uint32_t GetRadix() const { return m_radix; }
+  bool IsUnsigned() const { return m_is_unsigned; }
+  IntegerTypeSuffix GetTypeSuffix() const { return m_type; }
+
+  static bool classof(const ASTNode *node) {
+    return node->GetKind() == NodeKind::eIntegerLiteralNode;
+  }
+
+private:
+  llvm::APInt m_value;
+  uint32_t m_radix;
+  bool m_is_unsigned;
+  IntegerTypeSuffix m_type;
+};
+
+class FloatLiteralNode : public ASTNode {
+public:
+  FloatLiteralNode(uint32_t location, llvm::APFloat value)
+      : ASTNode(location, NodeKind::eFloatLiteralNode),
+        m_value(std::move(value)) {}
+
+  llvm::Expected<lldb::ValueObjectSP> Accept(Visitor *v) const override;
+
+  const llvm::APFloat &GetValue() const { return m_value; }
+
+  static bool classof(const ASTNode *node) {
+    return node->GetKind() == NodeKind::eFloatLiteralNode;
+  }
+
+private:
+  llvm::APFloat m_value;
+};
+
 /// This class contains one Visit method for each specialized type of
 /// DIL AST node. The Visit methods are used to dispatch a DIL AST node to
 /// the correct function in the DIL expression evaluator for evaluating that
@@ -195,6 +243,10 @@ class Visitor {
   Visit(const ArraySubscriptNode *node) = 0;
   virtual llvm::Expected<lldb::ValueObjectSP>
   Visit(const BitFieldExtractionNode *node) = 0;
+  virtual llvm::Expected<lldb::ValueObjectSP>
+  Visit(const IntegerLiteralNode *node) = 0;
+  virtual llvm::Expected<lldb::ValueObjectSP>
+  Visit(const FloatLiteralNode *node) = 0;
 };
 
 } // namespace lldb_private::dil

lldb/include/lldb/ValueObject/DILEval.h

@@ -54,6 +54,15 @@ class Interpreter : Visitor {
   Visit(const ArraySubscriptNode *node) override;
   llvm::Expected<lldb::ValueObjectSP>
   Visit(const BitFieldExtractionNode *node) override;
+  llvm::Expected<lldb::ValueObjectSP>
+  Visit(const IntegerLiteralNode *node) override;
+  llvm::Expected<lldb::ValueObjectSP>
+  Visit(const FloatLiteralNode *node) override;
+
+  llvm::Expected<CompilerType>
+  PickIntegerType(lldb::TypeSystemSP type_system,
+                  std::shared_ptr<ExecutionContextScope> ctx,
+                  const IntegerLiteralNode *literal);
 
   // Used by the interpreter to create objects, perform casts, etc.
   lldb::TargetSP m_target;

lldb/include/lldb/ValueObject/DILLexer.h

@@ -28,12 +28,14 @@ class Token {
     arrow,
     coloncolon,
     eof,
+    float_constant,
     identifier,
+    integer_constant,
     l_paren,
     l_square,
     minus,
-    numeric_constant,
     period,
+    plus,
     r_paren,
     r_square,
     star,

lldb/include/lldb/ValueObject/DILParser.h

@@ -96,6 +96,9 @@ class DILParser {
   std::string ParseIdExpression();
   std::string ParseUnqualifiedId();
   std::optional<int64_t> ParseIntegerConstant();
+  ASTNodeUP ParseNumericLiteral();
+  ASTNodeUP ParseIntegerLiteral();
+  ASTNodeUP ParseFloatingPointLiteral();
 
   void BailOut(const std::string &error, uint32_t loc, uint16_t err_len);
 

lldb/source/ValueObject/DILAST.cpp

@@ -37,4 +37,13 @@ BitFieldExtractionNode::Accept(Visitor *v) const {
   return v->Visit(this);
 }
 
+llvm::Expected<lldb::ValueObjectSP>
+IntegerLiteralNode::Accept(Visitor *v) const {
+  return v->Visit(this);
+}
+
+llvm::Expected<lldb::ValueObjectSP> FloatLiteralNode::Accept(Visitor *v) const {
+  return v->Visit(this);
+}
+
 } // namespace lldb_private::dil

lldb/source/ValueObject/DILEval.cpp

@@ -7,7 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "lldb/ValueObject/DILEval.h"
+#include "lldb/Core/Module.h"
 #include "lldb/Symbol/CompileUnit.h"
+#include "lldb/Symbol/TypeSystem.h"
 #include "lldb/Symbol/VariableList.h"
 #include "lldb/Target/RegisterContext.h"
 #include "lldb/ValueObject/DILAST.h"
@@ -497,4 +499,107 @@ Interpreter::Visit(const BitFieldExtractionNode *node) {
   return child_valobj_sp;
 }
 
+static llvm::Expected<lldb::TypeSystemSP>
+GetTypeSystemFromCU(std::shared_ptr<StackFrame> ctx) {
+  SymbolContext symbol_context =
+      ctx->GetSymbolContext(lldb::eSymbolContextCompUnit);
+  lldb::LanguageType language = symbol_context.comp_unit->GetLanguage();
+
+  symbol_context = ctx->GetSymbolContext(lldb::eSymbolContextModule);
+  return symbol_context.module_sp->GetTypeSystemForLanguage(language);
+}
+
+static CompilerType GetBasicType(lldb::TypeSystemSP type_system,
+                                 lldb::BasicType basic_type) {
+  if (type_system)
+    return type_system.get()->GetBasicTypeFromAST(basic_type);
+
+  return CompilerType();
+}
+
+llvm::Expected<CompilerType>
+Interpreter::PickIntegerType(lldb::TypeSystemSP type_system,
+                             std::shared_ptr<ExecutionContextScope> ctx,
+                             const IntegerLiteralNode *literal) {
+  // Binary, Octal, Hexadecimal and literals with a U suffix are allowed to be
+  // an unsigned integer.
+  bool unsigned_is_allowed = literal->IsUnsigned() || literal->GetRadix() != 10;
+  llvm::APInt apint = literal->GetValue();
+
+  llvm::SmallVector<std::pair<lldb::BasicType, lldb::BasicType>, 3> candidates;
+  if (literal->GetTypeSuffix() <= IntegerTypeSuffix::None)
+    candidates.emplace_back(lldb::eBasicTypeInt,
+                            unsigned_is_allowed ? lldb::eBasicTypeUnsignedInt
+                                                : lldb::eBasicTypeInvalid);
+  if (literal->GetTypeSuffix() <= IntegerTypeSuffix::Long)
+    candidates.emplace_back(lldb::eBasicTypeLong,
+                            unsigned_is_allowed ? lldb::eBasicTypeUnsignedLong
+                                                : lldb::eBasicTypeInvalid);
+  candidates.emplace_back(lldb::eBasicTypeLongLong,
+                          lldb::eBasicTypeUnsignedLongLong);
+  for (auto [signed_, unsigned_] : candidates) {
+    CompilerType signed_type = type_system->GetBasicTypeFromAST(signed_);
+    if (!signed_type)
+      continue;
+    llvm::Expected<uint64_t> size = signed_type.GetBitSize(ctx.get());
+    if (!size)
+      return size.takeError();
+    if (!literal->IsUnsigned() && apint.isIntN(*size - 1))
+      return signed_type;
+    if (unsigned_ != lldb::eBasicTypeInvalid && apint.isIntN(*size))
+      return type_system->GetBasicTypeFromAST(unsigned_);
+  }
+
+  return llvm::make_error<DILDiagnosticError>(
+      m_expr,
+      "integer literal is too large to be represented in any integer type",
+      literal->GetLocation());
+}
+
+llvm::Expected<lldb::ValueObjectSP>
+Interpreter::Visit(const IntegerLiteralNode *node) {
+  llvm::Expected<lldb::TypeSystemSP> type_system =
+      GetTypeSystemFromCU(m_exe_ctx_scope);
+  if (!type_system)
+    return type_system.takeError();
+
+  llvm::Expected<CompilerType> type =
+      PickIntegerType(*type_system, m_exe_ctx_scope, node);
+  if (!type)
+    return type.takeError();
+
+  Scalar scalar = node->GetValue();
+  // APInt from StringRef::getAsInteger comes with just enough bitwidth to
+  // hold the value. This adjusts APInt bitwidth to match the compiler type.
+  llvm::Expected<uint64_t> type_bitsize =
+      type->GetBitSize(m_exe_ctx_scope.get());
+  if (!type_bitsize)
+    return type_bitsize.takeError();
+  scalar.TruncOrExtendTo(*type_bitsize, false);
+  return ValueObject::CreateValueObjectFromScalar(m_target, scalar, *type,
+                                                  "result");
+}
+
+llvm::Expected<lldb::ValueObjectSP>
+Interpreter::Visit(const FloatLiteralNode *node) {
+  llvm::Expected<lldb::TypeSystemSP> type_system =
+      GetTypeSystemFromCU(m_exe_ctx_scope);
+  if (!type_system)
+    return type_system.takeError();
+
+  bool isFloat =
+      &node->GetValue().getSemantics() == &llvm::APFloat::IEEEsingle();
+  lldb::BasicType basic_type =
+      isFloat ? lldb::eBasicTypeFloat : lldb::eBasicTypeDouble;
+  CompilerType type = GetBasicType(*type_system, basic_type);
+
+  if (!type)
+    return llvm::make_error<DILDiagnosticError>(
+        m_expr, "unable to create a const literal", node->GetLocation());
+
+  Scalar scalar = node->GetValue();
+  return ValueObject::CreateValueObjectFromScalar(m_target, scalar, type,
+                                                  "result");
+}
+
 } // namespace lldb_private::dil

lldb/source/ValueObject/DILLexer.cpp

@@ -28,18 +28,23 @@ llvm::StringRef Token::GetTokenName(Kind kind) {
     return "coloncolon";
   case Kind::eof:
     return "eof";
+  case Kind::float_constant:
+    return "float_constant";
   case Kind::identifier:
     return "identifier";
+  case Kind::integer_constant:
+    return "integer_constant";
   case Kind::l_paren:
     return "l_paren";
   case Kind::l_square:
     return "l_square";
   case Kind::minus:
     return "minus";
-  case Kind::numeric_constant:
-    return "numeric_constant";
   case Kind::period:
     return "period";
+    return "l_square";
+  case Kind::plus:
+    return "plus";
   case Kind::r_paren:
     return "r_paren";
   case Kind::r_square:
@@ -70,13 +75,32 @@ static std::optional<llvm::StringRef> IsWord(llvm::StringRef expr,
   return candidate;
 }
 
-static bool IsNumberBodyChar(char ch) { return IsDigit(ch) || IsLetter(ch); }
+static bool IsNumberBodyChar(char ch) {
+  return IsDigit(ch) || IsLetter(ch) || ch == '.';
+}
 
-static std::optional<llvm::StringRef> IsNumber(llvm::StringRef expr,
-                                               llvm::StringRef &remainder) {
-  if (IsDigit(remainder[0])) {
-    llvm::StringRef number = remainder.take_while(IsNumberBodyChar);
-    remainder = remainder.drop_front(number.size());
+static std::optional<llvm::StringRef> IsNumber(llvm::StringRef &remainder,
+                                               bool &isFloat) {
+  llvm::StringRef tail = remainder;
+  llvm::StringRef body = tail.take_while(IsNumberBodyChar);
+  size_t dots = body.count('.');
+  if (dots > 1 || dots == body.size())
+    return std::nullopt;
+  if (IsDigit(body.front()) || (body[0] == '.' && IsDigit(body[1]))) {
+    isFloat = dots == 1;
+    tail = tail.drop_front(body.size());
+    bool isHex = body.contains_insensitive('x');
+    bool hasExp = !isHex && body.contains_insensitive('e');
+    bool hasHexExp = isHex && body.contains_insensitive('p');
+    if (hasExp || hasHexExp) {
+      isFloat = true; // This marks numbers like 0x1p1 and 1e1 as float
+      if (body.ends_with_insensitive("e") || body.ends_with_insensitive("p"))
+        if (tail.consume_front("+") || tail.consume_front("-"))
+          tail = tail.drop_while(IsNumberBodyChar);
+    }
+    size_t number_length = remainder.size() - tail.size();
+    llvm::StringRef number = remainder.take_front(number_length);
+    remainder = remainder.drop_front(number_length);
     return number;
   }
   return std::nullopt;
@@ -106,18 +130,21 @@ llvm::Expected<Token> DILLexer::Lex(llvm::StringRef expr,
     return Token(Token::eof, "", (uint32_t)expr.size());
 
   uint32_t position = cur_pos - expr.begin();
-  std::optional<llvm::StringRef> maybe_number = IsNumber(expr, remainder);
-  if (maybe_number)
-    return Token(Token::numeric_constant, maybe_number->str(), position);
+  bool isFloat = false;
+  std::optional<llvm::StringRef> maybe_number = IsNumber(remainder, isFloat);
+  if (maybe_number) {
+    auto kind = isFloat ? Token::float_constant : Token::integer_constant;
+    return Token(kind, maybe_number->str(), position);
+  }
   std::optional<llvm::StringRef> maybe_word = IsWord(expr, remainder);
   if (maybe_word)
     return Token(Token::identifier, maybe_word->str(), position);
 
   constexpr std::pair<Token::Kind, const char *> operators[] = {
-      {Token::amp, "&"},     {Token::arrow, "->"},   {Token::coloncolon, "::"},
-      {Token::l_paren, "("}, {Token::l_square, "["}, {Token::minus, "-"},
-      {Token::period, "."},  {Token::r_paren, ")"},  {Token::r_square, "]"},
-      {Token::star, "*"},
+      {Token::amp, "&"},      {Token::arrow, "->"},   {Token::coloncolon, "::"},
+      {Token::l_paren, "("},  {Token::l_square, "["}, {Token::minus, "-"},
+      {Token::period, "."},   {Token::plus, "+"},     {Token::r_paren, ")"},
+      {Token::r_square, "]"}, {Token::star, "*"},
   };
   for (auto [kind, str] : operators) {
     if (remainder.consume_front(str))