Skip to content

Commit 456da60

Browse files
markolbertmarkolbert
committed
LL(1) tokenizer implemented and appears to be working. Cleaned up/reorganized files.
1 parent 2a0ca02 commit 456da60

23 files changed

+888
-30
lines changed

‎Binder.Tests/Binder.Tests.csproj‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,9 @@
5353
<None Update="singleProperties.json">
5454
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
5555
</None>
56+
<None Update="tokenizer.json">
57+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
58+
</None>
5659
</ItemGroup>
5760

5861
</Project>

‎Binder.Tests/MiscAllocation.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@ public void StringHandling( string cmdLine, string? result )
2121
allocResult.UnknownKeys.Should().BeEmpty();
2222
allocResult.UnkeyedParameters.Should().BeEmpty();
2323

24-
option!.CommandLineValues.Count.Should().Be( 1 );
25-
option.CommandLineValues[ 0 ].Should().Be( result );
24+
option!.Values.Count.Should().Be( 1 );
25+
option.Values[ 0 ].Should().Be( result );
2626
}
2727
}
2828
}

‎Binder.Tests/new-alloc/TokenizerTests.cs‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using System.Text;
5+
using System.Threading.Tasks;
6+
using FluentAssertions;
7+
using J4JSoftware.CommandLine;
8+
using Xunit;
9+
10+
namespace J4JSoftware.Binder.Tests
11+
{
12+
public class TokenizerTests
13+
{
14+
[Theory]
15+
[MemberData(nameof(TestDataSource.GetTokenizerData), MemberType = typeof(TestDataSource))]
16+
public void Simple( TokenizerConfig config )
17+
{
18+
var cmdLogger = new CommandLineLogger();
19+
var tokenColl = TokenCollection.GetDefault( CommandLineStyle.Windows, cmdLogger );
20+
21+
var tokens = tokenColl.Tokenize( config.CommandLine );
22+
23+
tokens.Count.Should().Be( config.Data.Count + 1 );
24+
25+
for( var idx = 0; idx < tokens.Count; idx++ )
26+
{
27+
if( idx < tokens.Count - 1 )
28+
{
29+
tokens[ idx ].Type.Should().Be( config.Data[ idx ].Type );
30+
tokens[ idx ].Text.Should().Be( config.Data[ idx ].Text );
31+
}
32+
else
33+
{
34+
tokens[idx].Type.Should().Be(TokenType.EndOfInput);
35+
tokens[idx].Text.Should().Be(string.Empty);
36+
}
37+
}
38+
}
39+
}
40+
}

‎Binder.Tests/test-data/TestDataSource.cs‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
using System;
22
using System.Collections.Generic;
33
using System.IO;
4+
using System.Linq;
5+
using System.Text;
46
using System.Text.Json;
57
using System.Text.Json.Serialization;
68

@@ -10,25 +12,35 @@ public static class TestDataSource
1012
{
1113
public static IEnumerable<object[]> GetSinglePropertyData()
1214
{
13-
foreach ( var config in GetConfigurations("singleProperties.json") )
15+
foreach ( var config in GetConfigurations<TestConfig>("singleProperties.json") )
1416
{
1517
yield return new object[] { config };
1618
}
1719
}
1820

1921
public static IEnumerable<object[]> GetEmbeddedPropertyData()
2022
{
21-
foreach (var config in GetConfigurations("embeddedProperties.json"))
23+
foreach (var config in GetConfigurations<TestConfig>("embeddedProperties.json"))
2224
{
2325
yield return new object[] { config };
2426
}
2527
}
2628

27-
private static List<TestConfig> GetConfigurations( string jsonFile )
29+
public static IEnumerable<object[]> GetTokenizerData()
30+
{
31+
//yield return new object[] { GetConfigurations<TokenizerConfig>( "tokenizer.json" ).Last() };
32+
foreach (var config in GetConfigurations<TokenizerConfig>("tokenizer.json"))
33+
{
34+
yield return new object[] { config };
35+
}
36+
}
37+
38+
private static List<T> GetConfigurations<T>( string jsonFile )
39+
where T: class, new()
2840
{
2941
var text = File.ReadAllText(Path.Combine(Environment.CurrentDirectory, jsonFile));
3042

31-
return JsonSerializer.Deserialize<List<TestConfig>>(
43+
return JsonSerializer.Deserialize<List<T>>(
3244
text,
3345
new JsonSerializerOptions()
3446
{

‎Binder.Tests/test-data/TokenizerConfig.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
using System.Collections.Generic;
2+
3+
namespace J4JSoftware.Binder.Tests
4+
{
5+
public class TokenizerConfig
6+
{
7+
public List<TokenizerData> Data { get; set; }
8+
public string CommandLine { get; set; }
9+
}
10+
}

‎Binder.Tests/test-data/TokenizerData.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
using J4JSoftware.CommandLine;
2+
3+
namespace J4JSoftware.Binder.Tests
4+
{
5+
public class TokenizerData
6+
{
7+
public TokenType Type { get; set; }
8+
public string Text { get; set; }
9+
}
10+
}

‎Binder.Tests/tokenizer.json‎

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
[
2+
{
3+
"CommandLine": "/x",
4+
"Data": [
5+
{
6+
"Type": "KeyPrefix",
7+
"Text": "/"
8+
},
9+
{
10+
"Type": "Text",
11+
"Text": "x"
12+
}
13+
]
14+
},
15+
{
16+
"CommandLine": "/x abcd",
17+
"Data": [
18+
{
19+
"Type": "KeyPrefix",
20+
"Text": "/"
21+
},
22+
{
23+
"Type": "Text",
24+
"Text": "x"
25+
},
26+
{
27+
"Type": "Separator",
28+
"Text": " "
29+
},
30+
{
31+
"Type": "Text",
32+
"Text": "abcd"
33+
}
34+
]
35+
},
36+
{
37+
"CommandLine": "/x \"ab cd\"",
38+
"Data": [
39+
{
40+
"Type": "KeyPrefix",
41+
"Text": "/"
42+
},
43+
{
44+
"Type": "Text",
45+
"Text": "x"
46+
},
47+
{
48+
"Type": "Separator",
49+
"Text": " "
50+
},
51+
{
52+
"Type": "Text",
53+
"Text": "ab cd"
54+
}
55+
]
56+
},
57+
{
58+
"CommandLine": "/x \"'Now is the time for all good men'\"",
59+
"Data": [
60+
{
61+
"Type": "KeyPrefix",
62+
"Text": "/"
63+
},
64+
{
65+
"Type": "Text",
66+
"Text": "x"
67+
},
68+
{
69+
"Type": "Separator",
70+
"Text": " "
71+
},
72+
{
73+
"Type": "Text",
74+
"Text": "'Now is the time for all good men'"
75+
}
76+
]
77+
},
78+
{
79+
"CommandLine": "/x \"ab cd /y def",
80+
"Data": [
81+
{
82+
"Type": "KeyPrefix",
83+
"Text": "/"
84+
},
85+
{
86+
"Type": "Text",
87+
"Text": "x"
88+
},
89+
{
90+
"Type": "Separator",
91+
"Text": " "
92+
},
93+
{
94+
"Type": "Text",
95+
"Text": "ab cd /y def"
96+
}
97+
]
98+
},
99+
100+
{
101+
"CommandLine": "/ x",
102+
"Data": [
103+
{
104+
"Type": "KeyPrefix",
105+
"Text": "/"
106+
},
107+
{
108+
"Type": "Separator",
109+
"Text": " "
110+
},
111+
{
112+
"Type": "Text",
113+
"Text": "x"
114+
}
115+
]
116+
},
117+
{
118+
"CommandLine": "abcd",
119+
"Data": [
120+
{
121+
"Type": "Text",
122+
"Text": "abcd"
123+
}
124+
]
125+
},
126+
{
127+
"CommandLine": " abcd",
128+
"Data": [
129+
{
130+
"Type": "Separator",
131+
"Text": " "
132+
},
133+
{
134+
"Type": "Text",
135+
"Text": "abcd"
136+
}
137+
]
138+
},
139+
{
140+
"CommandLine": "/x /y abc /z abc def",
141+
"Data": [
142+
{
143+
"Type": "KeyPrefix",
144+
"Text": "/"
145+
},
146+
{
147+
"Type": "Text",
148+
"Text": "x"
149+
},
150+
{
151+
"Type": "Separator",
152+
"Text": " "
153+
},
154+
{
155+
"Type": "KeyPrefix",
156+
"Text": "/"
157+
},
158+
{
159+
"Type": "Text",
160+
"Text": "y"
161+
},
162+
{
163+
"Type": "Separator",
164+
"Text": " "
165+
},
166+
{
167+
"Type": "Text",
168+
"Text": "abc"
169+
},
170+
{
171+
"Type": "Separator",
172+
"Text": " "
173+
},
174+
{
175+
"Type": "KeyPrefix",
176+
"Text": "/"
177+
},
178+
{
179+
"Type": "Text",
180+
"Text": "z"
181+
},
182+
{
183+
"Type": "Separator",
184+
"Text": " "
185+
},
186+
{
187+
"Type": "Text",
188+
"Text": "abc"
189+
},
190+
{
191+
"Type": "Separator",
192+
"Text": " "
193+
},
194+
{
195+
"Type": "Text",
196+
"Text": "def"
197+
}
198+
]
199+
}
200+
]

‎Binder/Binder.csproj‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@
1414
</ItemGroup>
1515

1616
<ItemGroup>
17+
<Compile Remove="allocating\AllocationContext.cs" />
18+
<Compile Remove="allocating\AllocationElement.AllocationContext2.cs" />
19+
<Compile Remove="allocating\AllocationElement.cs" />
20+
<Compile Remove="allocating\NewAllocator.cs" />
1721
<Compile Remove="options\IContextKey.cs" />
1822
<Compile Remove="options\OptionConfiguration.cs" />
1923
<Compile Remove="options\OptionRequiredAttribute.cs" />

‎Binder/J4JCommandLineProvider.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ public override void Load()
3030

3131
case OptionStyle.SingleValued:
3232
if( option.NumValuesAllocated > 0)
33-
Set(option.ContextPath, option.CommandLineValues[0]);
33+
Set(option.ContextPath, option.Values[0]);
3434

3535
break;
3636

@@ -40,14 +40,14 @@ public override void Load()
4040
// collections), but they contain multiple string values from
4141
// allocating the command line
4242
if (option.NumValuesAllocated > 0)
43-
Set( option.ContextPath, string.Join( ", ", option.CommandLineValues ) );
43+
Set( option.ContextPath, string.Join( ", ", option.Values ) );
4444

4545
break;
4646

4747
case OptionStyle.Collection:
4848
for( var idx = 0; idx < option.NumValuesAllocated; idx++ )
4949
{
50-
Set( $"{option.ContextPath}:{idx}", option.CommandLineValues[ idx ] );
50+
Set( $"{option.ContextPath}:{idx}", option.Values[ idx ] );
5151
}
5252

5353
break;

0 commit comments

Comments
 (0)