Many changes, to address all the review comments:

- Remove "DIL" prefix from DILTokenKind	and DILToken.
- Change the token kind	from an	enum class to an enum inside the Token
  class.
- Use CamelCase	for all	the method names.
- Replace Token::SetValues method with assignments.
- Use a	StringRef, not std::string,  to hold the input string in the lexer.
- Update the lexer to lex all the tokens at one	time. Added two	new methods
  for this: LexAll and GetNextToken.
- Made some of the Lexer methods private.
- Replaces StringMap with StringSwitch for fast keyword lookups.
- Updated GetTokenName to directly return StringRefs; removed default case from
  switch statement.
- Cleaned up code format in IsLetter	& IsDigit.
- Updated IsWord too return an iterator	range containing the word (if any).
- Updated Lex function (now called by LexAll) to return	an llvm::Expected
  token; removed look_ahead checks; changed the	operator lexing	to use
  a vector of operators	(as suggested).
- Cleaned up LookAhead method, now that	we know	all tokens have	already	been
  lexed.
- Added	helper function	to unittests, to help check a sequence of tokens.
- Generally cleaned up the tests to deal with all the code changes.

Author: cmtice

lldb/include/lldb/ValueObject/DILLexer.h

@@ -10,6 +10,8 @@
 #define LLDB_VALUEOBJECT_DILLEXER_H_
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/iterator_range.h"
+#include "llvm/Support/Error.h"
 #include <cstdint>
 #include <limits.h>
 #include <memory>
@@ -20,94 +22,82 @@ namespace lldb_private {
 
 namespace dil {
 
-enum class TokenKind {
-  coloncolon,
-  eof,
-  identifier,
-  invalid,
-  kw_namespace,
-  l_paren,
-  none,
-  r_paren,
-  unknown,
-};
-
 /// Class defining the tokens generated by the DIL lexer and used by the
 /// DIL parser.
-class DILToken {
+class Token {
 public:
-  DILToken(dil::TokenKind kind, std::string spelling, uint32_t start)
+  enum Kind {
+    coloncolon,
+    eof,
+    identifier,
+    invalid,
+    kw_namespace,
+    l_paren,
+    none,
+    r_paren,
+    unknown,
+  };
+
+  Token(Kind kind, std::string spelling, uint32_t start)
       : m_kind(kind), m_spelling(spelling), m_start_pos(start) {}
 
-  DILToken() : m_kind(dil::TokenKind::none), m_spelling(""), m_start_pos(0) {}
+  Token() : m_kind(Kind::none), m_spelling(""), m_start_pos(0) {}
 
-  void setKind(dil::TokenKind kind) { m_kind = kind; }
-  dil::TokenKind getKind() const { return m_kind; }
+  void SetKind(Kind kind) { m_kind = kind; }
 
-  std::string getSpelling() const { return m_spelling; }
+  Kind GetKind() const { return m_kind; }
 
-  uint32_t getLength() const { return m_spelling.size(); }
+  std::string GetSpelling() const { return m_spelling; }
 
-  bool is(dil::TokenKind kind) const { return m_kind == kind; }
+  uint32_t GetLength() const { return m_spelling.size(); }
 
-  bool isNot(dil::TokenKind kind) const { return m_kind != kind; }
+  bool Is(Kind kind) const { return m_kind == kind; }
 
-  bool isOneOf(dil::TokenKind kind1, dil::TokenKind kind2) const {
-    return is(kind1) || is(kind2);
-  }
+  bool IsNot(Kind kind) const { return m_kind != kind; }
 
-  template <typename... Ts> bool isOneOf(dil::TokenKind kind, Ts... Ks) const {
-    return is(kind) || isOneOf(Ks...);
-  }
+  bool IsOneOf(Kind kind1, Kind kind2) const { return Is(kind1) || Is(kind2); }
 
-  uint32_t getLocation() const { return m_start_pos; }
-
-  void setValues(dil::TokenKind kind, std::string spelling, uint32_t start) {
-    m_kind = kind;
-    m_spelling = spelling;
-    m_start_pos = start;
+  template <typename... Ts> bool IsOneOf(Kind kind, Ts... Ks) const {
+    return Is(kind) || IsOneOf(Ks...);
   }
 
-  static const std::string getTokenName(dil::TokenKind kind);
+  uint32_t GetLocation() const { return m_start_pos; }
+
+  static llvm::StringRef GetTokenName(Kind kind);
 
 private:
-  dil::TokenKind m_kind;
+  Kind m_kind;
   std::string m_spelling;
   uint32_t m_start_pos; // within entire expression string
 };
 
 /// Class for doing the simple lexing required by DIL.
 class DILLexer {
 public:
-  DILLexer(llvm::StringRef dil_expr) : m_expr(dil_expr.str()) {
+  DILLexer(llvm::StringRef dil_expr) : m_expr(dil_expr) {
     m_cur_pos = m_expr.begin();
     // Use UINT_MAX to indicate invalid/uninitialized value.
     m_tokens_idx = UINT_MAX;
+    m_invalid_token = Token(Token::invalid, "", 0);
   }
 
-  bool Lex(DILToken &result, bool look_ahead = false);
-
-  bool Is_Word(std::string::iterator start, uint32_t &length);
-
-  uint32_t GetLocation() { return m_cur_pos - m_expr.begin(); }
-
-  /// Update 'result' with the other paremeter values, create a
-  /// duplicate token, and push the duplicate token onto the vector of
-  /// lexed tokens.
-  void UpdateLexedTokens(DILToken &result, dil::TokenKind tok_kind,
-                         std::string tok_str, uint32_t tok_pos);
+  llvm::Expected<bool> LexAll();
 
   /// Return the lexed token N+1 positions ahead of the 'current' token
   /// being handled by the DIL parser.
-  const DILToken &LookAhead(uint32_t N);
+  const Token &LookAhead(uint32_t N);
+
+  const Token &AcceptLookAhead(uint32_t N);
 
-  const DILToken &AcceptLookAhead(uint32_t N);
+  const Token &GetNextToken();
 
   /// Return the index for the 'current' token being handled by the DIL parser.
   uint32_t GetCurrentTokenIdx() { return m_tokens_idx; }
 
   /// Return the current token to be handled by the DIL parser.
-  DILToken &GetCurrentToken() { return m_lexed_tokens[m_tokens_idx]; }
+  const Token &GetCurrentToken() { return m_lexed_tokens[m_tokens_idx]; }
+
+  uint32_t NumLexedTokens() { return m_lexed_tokens.size(); }
 
   /// Update the index for the 'current' token, to point to the next lexed
   /// token.
@@ -130,23 +120,35 @@ class DILLexer {
     return true;
   }
 
+  uint32_t GetLocation() { return m_cur_pos - m_expr.begin(); }
+
 private:
+  llvm::Expected<Token> Lex();
+
+  llvm::iterator_range<llvm::StringRef::iterator> IsWord();
+
+  /// Update 'result' with the other paremeter values, create a
+  /// duplicate token, and push the duplicate token onto the vector of
+  /// lexed tokens.
+  void UpdateLexedTokens(Token &result, Token::Kind tok_kind,
+                         std::string tok_str, uint32_t tok_pos);
+
   // The input string we are lexing & parsing.
-  std::string m_expr;
+  llvm::StringRef m_expr;
 
   // The current position of the lexer within m_expr (the character position,
   // within the string, of the next item to be lexed).
-  std::string::iterator m_cur_pos;
+  llvm::StringRef::iterator m_cur_pos;
 
   // Holds all of the tokens lexed so far.
-  std::vector<DILToken> m_lexed_tokens;
+  std::vector<Token> m_lexed_tokens;
 
   // Index into m_lexed_tokens; indicates which token the DIL parser is
   // currently trying to parse/handle.
   uint32_t m_tokens_idx;
 
   // "invalid" token; to be returned by lexer when 'look ahead' fails.
-  DILToken m_invalid_token;
+  Token m_invalid_token;
 };
 
 } // namespace dil