Using uint8 as token type to reduce memory usage

Author: aisk

compiler/parser/expression_parsing.go

@@ -266,7 +266,7 @@ func (p *Parser) parseInfixExpression(left ast.Expression) ast.Expression {
 	preced := p.curPrecedence()
 
 	// prevent "* *" from being parsed
-	if p.curToken.Literal == token.Asterisk && p.peekToken.Literal == token.Asterisk {
+	if p.curToken.Literal == token.Asterisk.String() && p.peekToken.Literal == token.Asterisk.String() {
 		msg := fmt.Sprintf("unexpected %s Line: %d", p.curToken.Literal, p.peekToken.Line)
 		p.error = errors.InitError(msg, errors.UnexpectedTokenError)
 		return nil

compiler/parser/flow_control_parsing.go

@@ -74,15 +74,15 @@ func (p *Parser) parseCaseConditional(base ast.Expression) *ast.ConditionalExpre
 
 func (p *Parser) parseCaseCondition(base ast.Expression) *ast.InfixExpression {
 	first := p.parseExpression(precedence.Normal)
-	infix := newInfixExpression(base, token.Token{Type: token.Eq, Literal: token.Eq}, first)
+	infix := newInfixExpression(base, token.Token{Type: token.Eq, Literal: token.Eq.String()}, first)
 
 	for p.peekTokenIs(token.Comma) {
 		p.nextToken()
 		p.nextToken()
 
 		right := p.parseExpression(precedence.Normal)
-		rightInfix := newInfixExpression(base, token.Token{Type: token.Eq, Literal: token.Eq}, right)
-		infix = newInfixExpression(infix, token.Token{Type: token.Or, Literal: token.Or}, rightInfix)
+		rightInfix := newInfixExpression(base, token.Token{Type: token.Eq, Literal: token.Eq.String()}, right)
+		infix = newInfixExpression(infix, token.Token{Type: token.Or, Literal: token.Or.String()}, rightInfix)
 	}
 
 	return infix

compiler/token/token.go

@@ -1,7 +1,7 @@
 package token
 
 // Type is used to determine token type
-type Type string
+type Type uint8
 
 // Token is structure for identifying input stream of characters
 type Token struct {
@@ -12,78 +12,155 @@ type Token struct {
 
 // Literals
 const (
-	Illegal = "ILLEGAL"
-	EOF     = "EOF"
-
-	Constant         = "CONSTANT"
-	Ident            = "IDENT"
-	InstanceVariable = "INSTANCE_VAR"
-	Int              = "INT"
-	Float            = "FLOAT"
-	String           = "STRING"
-	Comment          = "COMMENT"
-
-	Assign   = "="
-	Plus     = "+"
-	PlusEq   = "+="
-	Minus    = "-"
-	MinusEq  = "-="
-	Bang     = "!"
-	Asterisk = "*"
-	Pow      = "**"
-	Slash    = "/"
-	Dot      = "."
-	And      = "&&"
-	Or       = "||"
-	OrEq     = "||="
-	Modulo   = "%"
-
-	LT   = "<"
-	LTE  = "<="
-	GT   = ">"
-	GTE  = ">="
-	COMP = "<=>"
-
-	Comma     = ","
-	Semicolon = ";"
-	Colon     = ":"
-	Bar       = "|"
-
-	LParen   = "("
-	RParen   = ")"
-	LBrace   = "{"
-	RBrace   = "}"
-	LBracket = "["
-	RBracket = "]"
-
-	Eq    = "=="
-	NotEq = "!="
-	Range = ".."
-
-	True     = "TRUE"
-	False    = "FALSE"
-	Null     = "Null"
-	If       = "IF"
-	ElsIf    = "ELSIF"
-	Else     = "ELSE"
-	Case     = "CASE"
-	When     = "WHEN"
-	Return   = "RETURN"
-	Next     = "NEXT"
-	Break    = "BREAK"
-	Def      = "DEF"
-	Self     = "SELF"
-	End      = "END"
-	While    = "WHILE"
-	Do       = "DO"
-	Yield    = "YIELD"
-	GetBlock = "GET_BLOCK"
-	Class    = "CLASS"
-	Module   = "MODULE"
-
-	ResolutionOperator = "::"
+	Illegal Type = iota
+	EOF
+
+	Constant
+	Ident
+	InstanceVariable
+	Int
+	Float
+	String
+	Comment
+
+	Assign
+	Plus
+	PlusEq
+	Minus
+	MinusEq
+	Bang
+	Asterisk
+	Pow
+	Slash
+	Dot
+	And
+	Or
+	OrEq
+	Modulo
+
+	LT
+	LTE
+	GT
+	GTE
+	COMP
+
+	Comma
+	Semicolon
+	Colon
+	Bar
+
+	LParen
+	RParen
+	LBrace
+	RBrace
+	LBracket
+	RBracket
+
+	Eq
+	NotEq
+	Range
+
+	True
+	False
+	Null
+	If
+	ElsIf
+	Else
+	Case
+	When
+	Return
+	Next
+	Break
+	Def
+	Self
+	End
+	While
+	Do
+	Yield
+	GetBlock
+	Class
+	Module
+
+	ResolutionOperator
 )
 
+var tokenMap = map[Type]string{
+	Illegal: "ILLEGAL",
+	EOF:     "EOF",
+
+	Constant:         "CONSTANT",
+	Ident:            "IDENT",
+	InstanceVariable: "INSTANCE_VAR",
+	Int:              "INT",
+	Float:            "FLOAT",
+	String:           "STRING",
+	Comment:          "COMMENT",
+
+	Assign:   "=",
+	Plus:     "+",
+	PlusEq:   "+=",
+	Minus:    "-",
+	MinusEq:  "-=",
+	Bang:     "!",
+	Asterisk: "*",
+	Pow:      "**",
+	Slash:    "/",
+	Dot:      ".",
+	And:      "&&",
+	Or:       "||",
+	OrEq:     "||=",
+	Modulo:   "%",
+
+	LT:   "<",
+	LTE:  "<=",
+	GT:   ">",
+	GTE:  ">=",
+	COMP: "<=>",
+
+	Comma:     ",",
+	Semicolon: ";",
+	Colon:     ":",
+	Bar:       "|",
+
+	LParen:   "(",
+	RParen:   ")",
+	LBrace:   "{",
+	RBrace:   "}",
+	LBracket: "[",
+	RBracket: "]",
+
+	Eq:    "==",
+	NotEq: "!=",
+	Range: "..",
+
+	True:     "TRUE",
+	False:    "FALSE",
+	Null:     "Null",
+	If:       "IF",
+	ElsIf:    "ELSIF",
+	Else:     "ELSE",
+	Case:     "CASE",
+	When:     "WHEN",
+	Return:   "RETURN",
+	Next:     "NEXT",
+	Break:    "BREAK",
+	Def:      "DEF",
+	Self:     "SELF",
+	End:      "END",
+	While:    "WHILE",
+	Do:       "DO",
+	Yield:    "YIELD",
+	GetBlock: "GET_BLOCK",
+	Class:    "CLASS",
+	Module:   "MODULE",
+
+	ResolutionOperator: "::",
+}
+
+func (typ Type) String() string {
+	return tokenMap[typ]
+}
+
 var keywords = map[string]Type{
 	"def":       Def,
 	"true":      True,

native/ripper/ripper.go

@@ -345,7 +345,7 @@ func convertLex(t token.Type) string {
 	case token.Slash:
 		s = "slash"
 	default:
-		s = strings.ToLower(string(t))
+		s = strings.ToLower(t.String())
 	}
 
 	return "on_" + s