Skip to content

Commit 032cb10

Browse files
KathleenDollardmhutch
KathleenDollard
authored and
mhutch
committed
Tokenizer now tracks the location of tokens
* Preprocessed tokens can be skipped without messing up the location of subsequent tokens. * Tokens from response files can have accurate location information. This will enable better error handling and diagramming.
1 parent 5ae588a commit 032cb10

File tree

9 files changed

+420
-314
lines changed

9 files changed

+420
-314
lines changed

‎src/System.CommandLine.Tests/ParserTests.cs

Lines changed: 32 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,21 @@ namespace System.CommandLine.Tests
1616
{
1717
public partial class ParserTests
1818
{
19+
// TODO: Update testing strategy if we use Location in equality. Some will break
20+
private readonly Location dummyLocation = new("", Location.Internal, -1, null);
21+
1922
private T GetValue<T>(ParseResult parseResult, CliOption<T> option)
2023
=> parseResult.GetValue(option);
2124

2225
private T GetValue<T>(ParseResult parseResult, CliArgument<T> argument)
2326
=> parseResult.GetValue(argument);
2427

28+
//[Fact]
29+
//public void FailureTest()
30+
//{
31+
// Assert.True(false);
32+
//}
33+
2534
[Fact]
2635
public void An_option_can_be_checked_by_object_instance()
2736
{
@@ -1447,10 +1456,10 @@ public void Command_argument_arity_can_be_a_fixed_value_greater_than_1()
14471456
.Tokens
14481457
.Should()
14491458
.BeEquivalentTo(
1450-
new CliToken("1", CliTokenType.Argument, argument),
1451-
new CliToken("2", CliTokenType.Argument, argument),
1452-
new CliToken("3", CliTokenType.Argument, argument));
1453-
}
1459+
new CliToken("1", CliTokenType.Argument, argument,dummyLocation),
1460+
new CliToken("2", CliTokenType.Argument, argument, dummyLocation),
1461+
new CliToken("3", CliTokenType.Argument, argument, dummyLocation));
1462+
}
14541463

14551464
[Fact]
14561465
public void Command_argument_arity_can_be_a_range_with_a_lower_bound_greater_than_1()
@@ -1469,19 +1478,19 @@ public void Command_argument_arity_can_be_a_range_with_a_lower_bound_greater_tha
14691478
.Tokens
14701479
.Should()
14711480
.BeEquivalentTo(
1472-
new CliToken("1", CliTokenType.Argument, argument),
1473-
new CliToken("2", CliTokenType.Argument, argument),
1474-
new CliToken("3", CliTokenType.Argument, argument));
1481+
new CliToken("1", CliTokenType.Argument, argument, dummyLocation),
1482+
new CliToken("2", CliTokenType.Argument, argument, dummyLocation),
1483+
new CliToken("3", CliTokenType.Argument, argument, dummyLocation));
14751484
CliParser.Parse(command, "1 2 3 4 5")
14761485
.CommandResult
14771486
.Tokens
14781487
.Should()
14791488
.BeEquivalentTo(
1480-
new CliToken("1", CliTokenType.Argument, argument),
1481-
new CliToken("2", CliTokenType.Argument, argument),
1482-
new CliToken("3", CliTokenType.Argument, argument),
1483-
new CliToken("4", CliTokenType.Argument, argument),
1484-
new CliToken("5", CliTokenType.Argument, argument));
1489+
new CliToken("1", CliTokenType.Argument, argument, dummyLocation),
1490+
new CliToken("2", CliTokenType.Argument, argument, dummyLocation),
1491+
new CliToken("3", CliTokenType.Argument, argument, dummyLocation),
1492+
new CliToken("4", CliTokenType.Argument, argument, dummyLocation),
1493+
new CliToken("5", CliTokenType.Argument, argument, dummyLocation));
14851494
}
14861495

14871496
// TODO: Validation?
@@ -1541,9 +1550,9 @@ public void Option_argument_arity_can_be_a_fixed_value_greater_than_1()
15411550
.Tokens
15421551
.Should()
15431552
.BeEquivalentTo(
1544-
new CliToken("1", CliTokenType.Argument, default),
1545-
new CliToken("2", CliTokenType.Argument, default),
1546-
new CliToken("3", CliTokenType.Argument, default));
1553+
new CliToken("1", CliTokenType.Argument, default, dummyLocation),
1554+
new CliToken("2", CliTokenType.Argument, default, dummyLocation),
1555+
new CliToken("3", CliTokenType.Argument, default, dummyLocation));
15471556
}
15481557

15491558
[Fact]
@@ -1561,19 +1570,19 @@ public void Option_argument_arity_can_be_a_range_with_a_lower_bound_greater_than
15611570
.Tokens
15621571
.Should()
15631572
.BeEquivalentTo(
1564-
new CliToken("1", CliTokenType.Argument, default),
1565-
new CliToken("2", CliTokenType.Argument, default),
1566-
new CliToken("3", CliTokenType.Argument, default));
1573+
new CliToken("1", CliTokenType.Argument, default, dummyLocation),
1574+
new CliToken("2", CliTokenType.Argument, default, dummyLocation),
1575+
new CliToken("3", CliTokenType.Argument, default, dummyLocation));
15671576
CliParser.Parse(command, "-x 1 -x 2 -x 3 -x 4 -x 5")
15681577
.GetResult(option)
15691578
.Tokens
15701579
.Should()
15711580
.BeEquivalentTo(
1572-
new CliToken("1", CliTokenType.Argument, default),
1573-
new CliToken("2", CliTokenType.Argument, default),
1574-
new CliToken("3", CliTokenType.Argument, default),
1575-
new CliToken("4", CliTokenType.Argument, default),
1576-
new CliToken("5", CliTokenType.Argument, default));
1581+
new CliToken("1", CliTokenType.Argument, default, dummyLocation),
1582+
new CliToken("2", CliTokenType.Argument, default, dummyLocation),
1583+
new CliToken("3", CliTokenType.Argument, default, dummyLocation),
1584+
new CliToken("4", CliTokenType.Argument, default, dummyLocation),
1585+
new CliToken("5", CliTokenType.Argument, default, dummyLocation));
15771586
}
15781587

15791588
// TODO: Validation?

‎src/System.CommandLine.Tests/TokenizerTests.cs

Lines changed: 57 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,14 @@ public partial class TokenizerTests
1515
{
1616

1717
[Fact]
18-
public void The_tokenizer_is_accessible()
18+
public void The_tokenizer_can_handle_single_option()
1919
{
2020
var option = new CliOption<string>("--hello");
2121
var command = new CliRootCommand { option };
2222
IReadOnlyList<string> args = ["--hello", "world"];
2323
List<CliToken> tokens = null;
2424
List<string> errors = null;
25-
CliTokenizer.Tokenize(args,command,false, true, out tokens, out errors);
25+
Tokenizer.Tokenize(args, command, new CliConfiguration(command), true, out tokens, out errors);
2626

2727
tokens
2828
.Skip(1)
@@ -32,5 +32,60 @@ public void The_tokenizer_is_accessible()
3232

3333
errors.Should().BeNull();
3434
}
35+
36+
[Fact]
37+
public void Location_stack_is_correct()
38+
{
39+
var option = new CliOption<string>("--hello");
40+
var command = new CliRootCommand { option };
41+
IReadOnlyList<string> args = ["--hello", "world"];
42+
List<CliToken> tokens = null;
43+
List<string> errors = null;
44+
45+
int rootCommandNameLength = CliExecutable.ExecutableName.Length;
46+
47+
Tokenizer.Tokenize(args,
48+
command,
49+
new CliConfiguration(command),
50+
true,
51+
out tokens,
52+
out errors);
53+
54+
var locations = tokens
55+
.Skip(1)
56+
.Select(t => t.Location.ToString())
57+
.ToList();
58+
errors.Should().BeNull();
59+
tokens.Count.Should().Be(3);
60+
locations.Count.Should().Be(2);
61+
locations[0].Should().Be($"User [-1, {rootCommandNameLength}, 0]; User [0, 7, 0]");
62+
locations[1].Should().Be($"User [-1, {rootCommandNameLength}, 0]; User [1, 5, 0]");
63+
}
64+
65+
[Fact]
66+
public void Directives_are_skipped()
67+
{
68+
var option = new CliOption<string>("--hello");
69+
var command = new CliRootCommand { option };
70+
var configuration = new CliConfiguration(command);
71+
configuration.AddPreprocessedLocation(new Location("[diagram]", Location.User, 0, null));
72+
IReadOnlyList<string> args = ["[diagram] --hello", "world"];
73+
74+
List<CliToken> tokens = null;
75+
List<string> errors = null;
76+
77+
Tokenizer.Tokenize(args,
78+
command,
79+
new CliConfiguration(command),
80+
true,
81+
out tokens,
82+
out errors);
83+
84+
var hasDiagram = tokens
85+
.Any(t => t.Value == "[diagram]");
86+
errors.Should().BeNull();
87+
tokens.Count.Should().Be(3); // root is a token
88+
hasDiagram .Should().BeFalse();
89+
}
3590
}
3691
}

‎src/System.CommandLine/CliConfiguration.cs

Lines changed: 42 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,45 @@ public CliConfiguration(CliCommand rootCommand)
5555
///
5656
/// </remarks>
5757
public bool EnablePosixBundling { get; set; } = true;
58+
59+
/// <summary>
60+
/// Indicates whether the first argument of the passed string is the exe name
61+
/// </summary>
62+
/// <param name="args">The args of a command line, such as those passed to Main(string[] args)</param>
63+
/// <returns></returns>
64+
// TODO: If this is the right model, tuck this away because it should only be used by subsystems.
65+
public bool FirstArgumentIsRootCommand(IReadOnlyList<string> args)
66+
{
67+
// TODO: This logic was previously that rawInput was null. Seems more sensible to look for an empty args array.From private static ParseResult Parse(CliCommand ,IReadOnlyList< string > ,string? ,CliConfiguration? ). CHeck logic and ensure test coverage
68+
return args.Any()
69+
? FirstArgLooksLikeRoot(args.First(), RootCommand)
70+
: false;
71+
72+
static bool FirstArgLooksLikeRoot(string firstArg, CliCommand rootCommand)
73+
{
74+
try
75+
{
76+
return firstArg == CliExecutable.ExecutablePath || rootCommand.EqualsNameOrAlias(Path.GetFileName(firstArg));
77+
}
78+
catch // possible exception for illegal characters in path on .NET Framework
79+
{
80+
return false;
81+
}
82+
83+
}
84+
}
85+
86+
private List<Location>? preprocessedLocations = null;
87+
public IEnumerable<Location>? PreProcessedLocations => preprocessedLocations;
88+
public void AddPreprocessedLocation(Location location)
89+
{
90+
if (preprocessedLocations is null)
91+
{
92+
preprocessedLocations = new List<Location>();
93+
}
94+
preprocessedLocations.Add(location);
95+
}
96+
5897
/*
5998
/// <summary>
6099
/// Enables a default exception handler to catch any unhandled exceptions thrown during invocation. Enabled by default.
@@ -67,6 +106,7 @@ public CliConfiguration(CliCommand rootCommand)
67106
/// If not provided, a default timeout of 2 seconds is enforced.
68107
/// </summary>
69108
public TimeSpan? ProcessTerminationTimeout { get; set; } = TimeSpan.FromSeconds(2);
109+
*/
70110

71111
/// <summary>
72112
/// Response file token replacer, enabled by default.
@@ -75,8 +115,8 @@ public CliConfiguration(CliCommand rootCommand)
75115
/// <remarks>
76116
/// When enabled, any token prefixed with <code>@</code> can be replaced with zero or more other tokens. This is mostly commonly used to expand tokens from response files and interpolate them into a command line prior to parsing.
77117
/// </remarks>
78-
public TryReplaceToken? ResponseFileTokenReplacer { get; set; } = StringExtensions.TryReadResponseFile;
79-
*/
118+
public Func<string, (List<string>? tokens, List<string>? errors)>? ResponseFileTokenReplacer { get; set; }
119+
80120
/// <summary>
81121
/// Gets the root command.
82122
/// </summary>

‎src/System.CommandLine/ParseResult.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ public sealed class ParseResult
2525

2626
internal ParseResult(
2727
CliConfiguration configuration,
28+
// TODO: determine how rootCommandResult and commandResult differ
2829
CommandResult rootCommandResult,
2930
CommandResult commandResult,
3031
List<CliToken> tokens,

‎src/System.CommandLine/Parsing/CliParser.cs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,7 @@ string CurrentToken()
138138
bool IsAtEndOfInput() => pos == memory.Length;
139139
}
140140

141+
// TODO: I'd like a name change where all refs to the string args passed to main are "args" and arguments refers to CLI arguments
141142
private static ParseResult Parse(
142143
CliCommand rootCommand,
143144
IReadOnlyList<string> arguments,
@@ -151,11 +152,11 @@ private static ParseResult Parse(
151152

152153
configuration ??= new CliConfiguration(rootCommand);
153154

154-
CliTokenizer.Tokenize(
155+
Tokenizer.Tokenize(
155156
arguments,
156157
rootCommand,
158+
configuration,
157159
inferRootCommand: rawInput is not null,
158-
configuration.EnablePosixBundling,
159160
out List<CliToken> tokens,
160161
out List<string>? tokenizationErrors);
161162

‎src/System.CommandLine/Parsing/CliToken.cs

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,42 +3,48 @@
33

44
namespace System.CommandLine.Parsing
55
{
6+
// TODO: Include location in equality
7+
68
// FIXME: should CliToken be public or internal? made internal for now
79
// FIXME: should CliToken be a struct?
810
/// <summary>
911
/// A unit of significant text on the command line.
1012
/// </summary>
1113
internal sealed class CliToken : IEquatable<CliToken>
1214
{
13-
internal const int ImplicitPosition = -1;
15+
public static CliToken CreateFromOtherToken(CliToken otherToken, string? arg, Location location)
16+
=> new(arg, otherToken.Type, otherToken.Symbol, location);
1417

1518
/// <param name="value">The string value of the token.</param>
1619
/// <param name="type">The type of the token.</param>
1720
/// <param name="symbol">The symbol represented by the token</param>
21+
/// <param name="location">The location of the token</param>
22+
/*
1823
public CliToken(string? value, CliTokenType type, CliSymbol symbol)
1924
{
2025
Value = value ?? "";
2126
Type = type;
2227
Symbol = symbol;
23-
Position = ImplicitPosition;
28+
Location = Location.CreateImplicit(value, value is null ? 0 : value.Length);
2429
}
25-
26-
internal CliToken(string? value, CliTokenType type, CliSymbol? symbol, int position)
30+
*/
31+
32+
internal CliToken(string? value, CliTokenType type, CliSymbol? symbol, Location location)
2733
{
2834
Value = value ?? "";
2935
Type = type;
3036
Symbol = symbol;
31-
Position = position;
37+
Location = location;
3238
}
3339

34-
internal int Position { get; }
40+
internal Location Location { get; }
3541

3642
/// <summary>
3743
/// The string value of the token.
3844
/// </summary>
3945
public string Value { get; }
4046

41-
internal bool Implicit => Position == ImplicitPosition;
47+
internal bool Implicit => Location.IsImplicit;
4248

4349
/// <summary>
4450
/// The type of the token.

‎src/System.CommandLine/Parsing/Location.cs

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
// Copyright (c) .NET Foundation and contributors. All rights reserved.
2+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
3+
4+
using System.Collections;
5+
using System.Collections.Generic;
6+
using System.ComponentModel;
7+
using static System.Net.Mime.MediaTypeNames;
8+
9+
namespace System.CommandLine.Parsing
10+
{
11+
public record Location
12+
{
13+
public const string Implicit = "Implicit";
14+
public const string Internal = "Internal";
15+
public const string User = "User";
16+
public const string Response = "Response";
17+
18+
internal static Location CreateRoot(string exeName, bool isImplicit, int start)
19+
=> new(exeName, isImplicit ? Internal : User, start, null);
20+
internal static Location CreateImplicit(string text, Location outerLocation, int offset = 0)
21+
=> new(text, Implicit, -1, outerLocation, offset);
22+
internal static Location CreateInternal(string text, Location? outerLocation = null, int offset = 0)
23+
=> new(text, Internal, -1, outerLocation, offset);
24+
internal static Location CreateUser(string text, int start, Location outerLocation, int offset = 0)
25+
=> new(text, User, start, outerLocation, offset);
26+
internal static Location CreateResponse(string responseSourceName, int start, Location outerLocation, int offset = 0)
27+
=> new(responseSourceName, $"{Response}:{responseSourceName}", start, outerLocation, offset);
28+
29+
internal static Location FromOuterLocation(string text, int start, Location outerLocation, int offset = 0)
30+
=> new(text, outerLocation.Source, start, outerLocation, offset);
31+
32+
public Location(string text, string source, int start, Location? outerLocation, int offset = 0)
33+
{
34+
Text = text;
35+
Source = source;
36+
Start = start;
37+
Length = text.Length;
38+
Offset = offset;
39+
OuterLocation = outerLocation;
40+
}
41+
42+
public string Text { get; }
43+
public string Source { get; }
44+
public int Start { get; }
45+
public int Offset { get; }
46+
public int Length { get; }
47+
public Location? OuterLocation { get; }
48+
49+
public bool IsImplicit
50+
=> Source == Implicit;
51+
52+
public override string ToString()
53+
=> $"{(OuterLocation is null ? "" : OuterLocation.ToString() + "; ")}{Source} [{Start}, {Length}, {Offset}]";
54+
55+
}
56+
}

0 commit comments

Comments
 (0)