Skip to content

Commit a3c4e65

Browse files
timmi-on-railstimmi-on-rails
committed
added integer/float difference in tokenizer
1 parent 90ab5de commit a3c4e65

File tree

11 files changed

+71
-29
lines changed

11 files changed

+71
-29
lines changed

‎ExpressionTest/TokenizerTest.cs‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ public void TestEveryTokenType()
3131
AssertTokensMatch(">=", Token(TokenType.GreaterOrEqual, ">=", 0), Token(TokenType.EndOfFile, "", 2));
3232
AssertTokensMatch("true", Token(TokenType.True, "true", 0), Token(TokenType.EndOfFile, "", 4));
3333
AssertTokensMatch("false", Token(TokenType.False, "false", 0), Token(TokenType.EndOfFile, "", 5));
34-
AssertTokensMatch("365", Token(TokenType.Numeric, "365", 0), Token(TokenType.EndOfFile, "", 3));
34+
AssertTokensMatch("365", Token(TokenType.Integer, "365", 0), Token(TokenType.EndOfFile, "", 3));
3535
AssertTokensMatch("x", Token(TokenType.Identifier, "x", 0), Token(TokenType.EndOfFile, "", 1));
3636
AssertTokensMatch("!", Token(TokenType.Exclamation, "!", 0), Token(TokenType.EndOfFile, "", 1));
3737
AssertTokensMatch(" ", Token(TokenType.WhiteSpace, " ", 0), Token(TokenType.EndOfFile, "", 2));
@@ -41,26 +41,26 @@ public void TestEveryTokenType()
4141
[Test]
4242
public void TestNumberToken()
4343
{
44-
AssertTokensMatch("23123", Token(TokenType.Numeric, "23123", 0), Token(TokenType.EndOfFile, "", 5));
45-
AssertTokensMatch(".23", Token(TokenType.Numeric, ".23", 0), Token(TokenType.EndOfFile, "", 3));
46-
AssertTokensMatch("13.3", Token(TokenType.Numeric, "13.3", 0), Token(TokenType.EndOfFile, "", 4));
44+
AssertTokensMatch("23123", Token(TokenType.Integer, "23123", 0), Token(TokenType.EndOfFile, "", 5));
45+
AssertTokensMatch(".23", Token(TokenType.FloatingPointNumber, ".23", 0), Token(TokenType.EndOfFile, "", 3));
46+
AssertTokensMatch("13.3", Token(TokenType.FloatingPointNumber, "13.3", 0), Token(TokenType.EndOfFile, "", 4));
4747
AssertTokensMatch("27.", Token(TokenType.Unknown, "27.", 0), Token(TokenType.EndOfFile, "", 3));
48-
AssertTokensMatch("1e10", Token(TokenType.Numeric, "1e10", 0), Token(TokenType.EndOfFile, "", 4));
48+
AssertTokensMatch("1e10", Token(TokenType.FloatingPointNumber, "1e10", 0), Token(TokenType.EndOfFile, "", 4));
4949
AssertTokensMatch("1.e10", Token(TokenType.Unknown, "1.e10", 0), Token(TokenType.EndOfFile, "", 5));
50-
AssertTokensMatch("1.7e3", Token(TokenType.Numeric, "1.7e3", 0), Token(TokenType.EndOfFile, "", 5));
51-
AssertTokensMatch("1.22e+4", Token(TokenType.Numeric, "1.22e+4", 0), Token(TokenType.EndOfFile, "", 7));
52-
AssertTokensMatch("1.22e-34", Token(TokenType.Numeric, "1.22e-34", 0), Token(TokenType.EndOfFile, "", 8));
50+
AssertTokensMatch("1.7e3", Token(TokenType.FloatingPointNumber, "1.7e3", 0), Token(TokenType.EndOfFile, "", 5));
51+
AssertTokensMatch("1.22e+4", Token(TokenType.FloatingPointNumber, "1.22e+4", 0), Token(TokenType.EndOfFile, "", 7));
52+
AssertTokensMatch("1.22e-34", Token(TokenType.FloatingPointNumber, "1.22e-34", 0), Token(TokenType.EndOfFile, "", 8));
5353
AssertTokensMatch("-3.4",
5454
Token(TokenType.Minus, "-", 0),
55-
Token(TokenType.Numeric, "3.4", 1),
55+
Token(TokenType.FloatingPointNumber, "3.4", 1),
5656
Token(TokenType.EndOfFile, "", 4));
5757
}
5858

5959
[Test]
6060
public void TestIdentifier()
6161
{
6262
AssertTokensMatch("43xyz",
63-
Token(TokenType.Numeric, "43", 0),
63+
Token(TokenType.Integer, "43", 0),
6464
Token(TokenType.Identifier, "xyz", 2),
6565
Token(TokenType.EndOfFile, "", 5));
6666
}
@@ -69,16 +69,16 @@ public void TestIdentifier()
6969
public void TestSomeOperators()
7070
{
7171
AssertTokensMatch("1+ y",
72-
Token(TokenType.Numeric, "1", 0),
72+
Token(TokenType.Integer, "1", 0),
7373
Token(TokenType.Plus, "+", 1),
7474
Token(TokenType.WhiteSpace, " ", 2),
7575
Token(TokenType.Identifier, "y", 4),
7676
Token(TokenType.EndOfFile, "", 5));
7777

7878
AssertTokensMatch("1<=2",
79-
Token(TokenType.Numeric, "1", 0),
79+
Token(TokenType.Integer, "1", 0),
8080
Token(TokenType.LessOrEqual, "<=", 1),
81-
Token(TokenType.Numeric, "2", 3),
81+
Token(TokenType.Integer, "2", 3),
8282
Token(TokenType.EndOfFile, "", 4));
8383
}
8484

‎shunting_yard/MathParser.csproj‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@
7575
<Compile Include="src\parser\parselets\BinaryOperatorParselet.cs" />
7676
<Compile Include="src\parser\parselets\GroupParselet.cs" />
7777
<Compile Include="src\parser\parselets\TernaryParselet.cs" />
78-
<Compile Include="src\parser\parselets\NumberParselet.cs" />
78+
<Compile Include="src\parser\parselets\IntegerParselet.cs" />
7979
<Compile Include="src\parser\parselets\VariableParselet.cs" />
8080
<Compile Include="src\parser\parselets\CallParselet.cs" />
8181
<Compile Include="src\parser\exceptions\BadAssignmentException.cs" />
@@ -91,6 +91,7 @@
9191
<Compile Include="src\expressions\Value.cs" />
9292
<Compile Include="src\tokenizer\Identifier.cs" />
9393
<Compile Include="src\parser\parselets\FixValueParselet.cs" />
94+
<Compile Include="src\parser\parselets\FloatingPointNumberParselet %28copy%29.cs" />
9495
</ItemGroup>
9596
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
9697
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.

‎shunting_yard/src/expressions/FunctionAssignmentExpression.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,15 @@ namespace MathParser
55
{
66
class FunctionAssignmentExpression : IExpression
77
{
8-
public string FunctionName { get; }
8+
public Identifier FunctionIdentifier { get; }
99

1010
public IEnumerable<Identifier> ArgumentNames { get; }
1111

1212
public IExpression Expression { get; }
1313

14-
public FunctionAssignmentExpression(string functionName, IEnumerable<Identifier> arguments, IExpression expression)
14+
public FunctionAssignmentExpression(Identifier functionIdentifier, IEnumerable<Identifier> arguments, IExpression expression)
1515
{
16-
FunctionName = functionName;
16+
FunctionIdentifier = functionIdentifier;
1717
Expression = expression;
1818
ArgumentNames = arguments.ToArray();
1919
}

‎shunting_yard/src/parser/ExpressionParser.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@ class ExpressionParser
1111
static ExpressionParser()
1212
{
1313
registerPrefixParselet(TokenType.Identifier, new VariableParselet());
14-
registerPrefixParselet(TokenType.Numeric, new NumberParselet());
14+
registerPrefixParselet(TokenType.FloatingPointNumber, new FloatingPointNumberParselet());
15+
registerPrefixParselet(TokenType.Integer, new IntegerParselet());
1516
registerPrefixParselet(TokenType.False, new FixValueParselet(Value.Boolean(false)));
1617
registerPrefixParselet(TokenType.True, new FixValueParselet(Value.Boolean(true)));
1718
registerPrefixParselet(TokenType.LeftParenthesis, new GroupParselet());

‎shunting_yard/src/parser/parselets/AssignParselet.cs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,12 @@ public IExpression Parse(ParseExpressionDelegate parseExpression, TokenStream to
1818
{
1919
CallExpression callExpression = (CallExpression)leftExpression;
2020

21-
IEnumerable<VariableExpression> arguments = callExpression.Arguments.Concat(new[] { callExpression.FunctionExpression }).Select(argument => (argument as VariableExpression));
21+
IEnumerable<VariableExpression> arguments = callExpression.Arguments.Select(argument => (argument as VariableExpression));
22+
VariableExpression functionExpression = callExpression.FunctionExpression as VariableExpression;
2223

23-
if (arguments.All(arg => arg != null))
24+
if (arguments.All(arg => arg != null) && functionExpression != null)
2425
{
25-
return new FunctionAssignmentExpression(((VariableExpression)callExpression.FunctionExpression).Identifier, arguments.Select(argument => argument.Identifier), rightExpression);
26+
return new FunctionAssignmentExpression(functionExpression.Identifier, arguments.Select(argument => argument.Identifier), rightExpression);
2627
}
2728
else
2829
{

‎shunting_yard/src/parser/parselets/NumberParselet.cs‎ renamed to ‎shunting_yard/src/parser/parselets/FloatingPointNumberParselet (copy).cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
namespace MathParser
55
{
6-
class NumberParselet : IPrefixParselet
6+
class FloatingPointNumberParselet : IPrefixParselet
77
{
88
public IExpression Parse(ParseExpressionDelegate parseExpression, TokenStream tokenStream, Token token)
99
{

‎shunting_yard/src/parser/parselets/IntegerParselet.cs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
using System;
2+
using System.Globalization;
3+
4+
namespace MathParser
5+
{
6+
class IntegerParselet : IPrefixParselet
7+
{
8+
public IExpression Parse(ParseExpressionDelegate parseExpression, TokenStream tokenStream, Token token)
9+
{
10+
long result = Int64.Parse(token.Content, NumberStyles.Any, CultureInfo.InvariantCulture);
11+
return new ValueExpression(Value.Integer(result));
12+
}
13+
}
14+
}

‎shunting_yard/src/tokenizer/TokenType.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@ enum TokenType
1313
Slash,
1414
Pow,
1515
Identifier,
16-
Numeric,
16+
Integer,
17+
FloatingPointNumber,
1718
QuestionMark,
1819
Colon,
1920
EndOfFile,

‎shunting_yard/src/tokenizer/Tokenizer.cs‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -139,10 +139,13 @@ IEnumerable<Token> Scan()
139139
if (IsWhiteSpace())
140140
{
141141
yield return ScanWhiteSpace();
142-
}
143-
else if (IsDigit() || Peek() == '.')
142+
}
143+
else if (IsDigit())
144+
{
145+
yield return ScanInteger();
146+
} else if (Peek() == '.')
144147
{
145-
yield return ScanNumber();
148+
yield return ScanFloatingPointNumber();
146149
}
147150
else if (IsLetter() || Peek() == '_')
148151
{
@@ -182,7 +185,22 @@ Token ScanIdentifier()
182185
return CreateToken(TokenType.Identifier);
183186
}
184187

185-
Token ScanNumber()
188+
Token ScanInteger()
189+
{
190+
while (IsDigit())
191+
{
192+
Consume();
193+
}
194+
195+
if (Peek() == '.' || Peek() == 'e')
196+
{
197+
return ScanFloatingPointNumber();
198+
}
199+
200+
return CreateToken(TokenType.Integer);
201+
}
202+
203+
Token ScanFloatingPointNumber()
186204
{
187205
while (IsDigit())
188206
{
@@ -222,7 +240,7 @@ Token ScanNumber()
222240
}
223241
}
224242

225-
return CreateToken(TokenType.Numeric);
243+
return CreateToken(TokenType.FloatingPointNumber);
226244
}
227245

228246
Token ScanPunctuation()

‎shunting_yard/src/visitors/AssignVisitor.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ public override void Visit(VariableAssignmentExpression variableAssignmentExpres
2121

2222
public override void Visit(FunctionAssignmentExpression functionAssignmentExpression)
2323
{
24-
_symbolManager.Set(functionAssignmentExpression.FunctionName, GetFunction(functionAssignmentExpression, _symbolManager));
24+
_symbolManager.Set(functionAssignmentExpression.FunctionIdentifier, GetFunction(functionAssignmentExpression, _symbolManager));
2525
}
2626

2727
public static Value GetFunction(FunctionAssignmentExpression functionAssignmentExpression, ISymbolManager symbolManager)

0 commit comments

Comments
 (0)