Fix #1665 (#1714)

* remove `ResponseFileHandling` enum and `TokenizeError` class

* remove a number of public setters / getters on `CommandLineBuilder` for tokenizer-level settings

* introduce `CommandLineBuilder.UseTokenReplacer` API

* generalize token replacement, implement response files as the default

Author: jonsequitur

src/System.CommandLine.ApiCompatibility.Tests/ApiCompatibilityApprovalTests.System_CommandLine_api_is_not_changed.approved.txt

@@ -71,10 +71,11 @@
     public static System.CommandLine.Parsing.ParseResult Parse(this Command command, System.String[] args)
     public static System.CommandLine.Parsing.ParseResult Parse(this Command command, System.String commandLine)
   public class CommandLineConfiguration
-    .ctor(Command command, System.Boolean enablePosixBundling = True, System.Boolean enableDirectives = True, System.Boolean enableLegacyDoubleDashBehavior = False, LocalizationResources resources = null, System.CommandLine.Parsing.ResponseFileHandling responseFileHandling = ParseArgsAsLineSeparated, System.Collections.Generic.IReadOnlyList<System.CommandLine.Invocation.InvocationMiddleware> middlewarePipeline = null, System.Func<System.CommandLine.Binding.BindingContext,System.CommandLine.Help.HelpBuilder> helpBuilderFactory = null)
+    .ctor(Command command, System.Boolean enablePosixBundling = True, System.Boolean enableDirectives = True, System.Boolean enableLegacyDoubleDashBehavior = False, System.Boolean enableTokenReplacement = True, LocalizationResources resources = null, System.Collections.Generic.IReadOnlyList<System.CommandLine.Invocation.InvocationMiddleware> middlewarePipeline = null, System.Func<System.CommandLine.Binding.BindingContext,System.CommandLine.Help.HelpBuilder> helpBuilderFactory = null, System.CommandLine.Parsing.TryReplaceToken tokenReplacer = null)
     public System.Boolean EnableDirectives { get; }
     public System.Boolean EnableLegacyDoubleDashBehavior { get; }
     public System.Boolean EnablePosixBundling { get; }
+    public System.Boolean EnableTokenReplacement { get; }
     public LocalizationResources LocalizationResources { get; }
     public Command RootCommand { get; }
     public System.Void ThrowIfInvalid()
@@ -250,10 +251,6 @@ System.CommandLine.Builder
   public class CommandLineBuilder
     .ctor(System.CommandLine.Command rootCommand = null)
     public System.CommandLine.Command Command { get; }
-    public System.Boolean EnableDirectives { get; set; }
-    public System.Boolean EnableLegacyDoubleDashBehavior { get; set; }
-    public System.Boolean EnablePosixBundling { get; set; }
-    public System.CommandLine.Parsing.ResponseFileHandling ResponseFileHandling { get; set; }
     public System.CommandLine.Parsing.Parser Build()
   public static class CommandLineBuilderExtensions
     public static CommandLineBuilder AddMiddleware(this CommandLineBuilder builder, System.CommandLine.Invocation.InvocationMiddleware middleware, System.CommandLine.Invocation.MiddlewareOrder order = Default)
@@ -262,7 +259,6 @@ System.CommandLine.Builder
     public static CommandLineBuilder EnableDirectives(this CommandLineBuilder builder, System.Boolean value = True)
     public static CommandLineBuilder EnableLegacyDoubleDashBehavior(this CommandLineBuilder builder, System.Boolean value = True)
     public static CommandLineBuilder EnablePosixBundling(this CommandLineBuilder builder, System.Boolean value = True)
-    public static CommandLineBuilder ParseResponseFileAs(this CommandLineBuilder builder, System.CommandLine.Parsing.ResponseFileHandling responseFileHandling)
     public static CommandLineBuilder RegisterWithDotnetSuggest(this CommandLineBuilder builder)
     public static CommandLineBuilder UseDefaults(this CommandLineBuilder builder)
     public static CommandLineBuilder UseEnvironmentVariableDirective(this CommandLineBuilder builder)
@@ -275,6 +271,7 @@ System.CommandLine.Builder
     public static CommandLineBuilder UseParseDirective(this CommandLineBuilder builder, System.Nullable<System.Int32> errorExitCode = null)
     public static CommandLineBuilder UseParseErrorReporting(this CommandLineBuilder builder, System.Nullable<System.Int32> errorExitCode = null)
     public static CommandLineBuilder UseSuggestDirective(this CommandLineBuilder builder)
+    public static CommandLineBuilder UseTokenReplacer(this CommandLineBuilder builder, System.CommandLine.Parsing.TryReplaceToken replaceToken)
     public static CommandLineBuilder UseTypoCorrections(this CommandLineBuilder builder, System.Int32 maxLevenshteinDistance = 3)
     public static CommandLineBuilder UseVersionOption(this CommandLineBuilder builder)
     public static CommandLineBuilder UseVersionOption(this CommandLineBuilder builder, System.String[] aliases)
@@ -476,10 +473,6 @@ System.CommandLine.Parsing
     public static System.Threading.Tasks.Task<System.Int32> InvokeAsync(this Parser parser, System.String commandLine, System.CommandLine.IConsole console = null)
     public static System.Threading.Tasks.Task<System.Int32> InvokeAsync(this Parser parser, System.String[] args, System.CommandLine.IConsole console = null)
     public static ParseResult Parse(this Parser parser, System.String commandLine)
-  public enum ResponseFileHandling : System.Enum, System.IComparable, System.IConvertible, System.IFormattable
-    ParseArgsAsLineSeparated=0
-    ParseArgsAsSpaceSeparated=1
-    Disabled=2
   public abstract class SymbolResult
     public System.Collections.Generic.IReadOnlyList<SymbolResult> Children { get; }
     public System.String ErrorMessage { get; set; }
@@ -505,16 +498,18 @@ System.CommandLine.Parsing
     public System.Boolean Equals(Token other)
     public System.Int32 GetHashCode()
     public System.String ToString()
-  public class TokenizeError
-    public System.String Message { get; }
-    public System.String ToString()
   public enum TokenType : System.Enum, System.IComparable, System.IConvertible, System.IFormattable
     Argument=0
     Command=1
     Option=2
     DoubleDash=3
     Unparsed=4
     Directive=5
+  public delegate TryReplaceToken : System.MulticastDelegate, System.ICloneable, System.Runtime.Serialization.ISerializable
+    .ctor(System.Object object, System.IntPtr method)
+    public System.IAsyncResult BeginInvoke(System.String tokenToReplace, ref System.Collections.Generic.IReadOnlyList<System.String> replacementTokens, ref System.String& errorMessage, System.AsyncCallback callback, System.Object object)
+    public System.Boolean EndInvoke(ref System.Collections.Generic.IReadOnlyList<System.String> replacementTokens, ref System.String& errorMessage, System.IAsyncResult result)
+    public System.Boolean Invoke(System.String tokenToReplace, ref System.Collections.Generic.IReadOnlyList<System.String> replacementTokens, ref System.String& errorMessage)
   public delegate ValidateSymbolResult<in T> : System.MulticastDelegate, System.ICloneable, System.Runtime.Serialization.ISerializable
     .ctor(System.Object object, System.IntPtr method)
     public System.IAsyncResult BeginInvoke(T symbolResult, System.AsyncCallback callback, System.Object object)

src/System.CommandLine.Tests/ResponseFileTests.cs

@@ -24,7 +24,7 @@ public void Dispose()
             }
         }
 
-        private string ResponseFile(params string[] lines)
+        private string CreateResponseFile(params string[] lines)
         {
             var responseFile = new FileInfo(Path.GetTempFileName());
 
@@ -46,15 +46,15 @@ public void When_response_file_specified_it_loads_options_from_response_file()
         {
             var option = new Option<bool>("--flag");
 
-            var result = option.Parse($"@{ResponseFile("--flag")}");
+            var result = option.Parse($"@{CreateResponseFile("--flag")}");
 
             result.HasOption(option).Should().BeTrue();
         }
 
         [Fact]
         public void When_response_file_is_specified_it_loads_options_with_arguments_from_response_file()
         {
-            var responseFile = ResponseFile(
+            var responseFile = CreateResponseFile(
                 "--flag",
                 "--flag2",
                 "123");
@@ -77,7 +77,7 @@ public void When_response_file_is_specified_it_loads_options_with_arguments_from
         [Fact]
         public void When_response_file_is_specified_it_loads_command_arguments_from_response_file()
         {
-            var responseFile = ResponseFile(
+            var responseFile = CreateResponseFile(
                 "one",
                 "two",
                 "three");
@@ -98,7 +98,7 @@ public void When_response_file_is_specified_it_loads_command_arguments_from_resp
         [Fact]
         public void Response_file_can_provide_subcommand_arguments()
         {
-            var responseFile = ResponseFile(
+            var responseFile = CreateResponseFile(
                 "one",
                 "two",
                 "three");
@@ -122,7 +122,7 @@ public void Response_file_can_provide_subcommand_arguments()
         [Fact]
         public void Response_file_can_provide_subcommand()
         {
-            var responseFile = ResponseFile("subcommand");
+            var responseFile = CreateResponseFile("subcommand");
 
             var result = new RootCommand
                          {
@@ -143,7 +143,7 @@ public void Response_file_can_provide_subcommand()
         [Fact]
         public void When_response_file_is_specified_it_loads_subcommand_arguments_from_response_file()
         {
-            var responseFile = ResponseFile(
+            var responseFile = CreateResponseFile(
                 "one",
                 "two",
                 "three");
@@ -167,7 +167,7 @@ public void When_response_file_is_specified_it_loads_subcommand_arguments_from_r
         [Fact]
         public void Response_file_can_contain_blank_lines()
         {
-            var responseFile = ResponseFile(
+            var responseFile = CreateResponseFile(
                 "--flag",
                 "",
                 "123");
@@ -190,7 +190,7 @@ public void Response_file_can_contain_comments_which_are_ignored_when_loaded()
             var optionOne = new Option<bool>("--flag");
             var optionTwo = new Option<bool>("--flag2");
 
-            var responseFile = ResponseFile(
+            var responseFile = CreateResponseFile(
                 "# comment one",
                 "--flag",
                 "# comment two",
@@ -278,7 +278,7 @@ public void When_response_file_cannot_be_read_then_specified_error_is_returned()
         [InlineData("--flag=\"first value\" --flag2=123")]
         public void When_response_file_parse_as_space_separated_returns_expected_values(string input)
         {
-            var responseFile = ResponseFile(input);
+            var responseFile = CreateResponseFile(input);
 
             var optionOne = new Option<string>("--flag");
             var optionTwo = new Option<int>("--flag2");
@@ -289,7 +289,6 @@ public void When_response_file_parse_as_space_separated_returns_expected_values(
                 optionTwo
             };
             var parser = new CommandLineBuilder(rootCommand)
-                         .ParseResponseFileAs(ResponseFileHandling.ParseArgsAsSpaceSeparated)
                          .Build();
 
             var result = parser.Parse($"@{responseFile}");
@@ -307,7 +306,8 @@ public void When_response_file_processing_is_disabled_then_it_returns_response_f
             };
             var configuration = new CommandLineConfiguration(
                 command,
-                responseFileHandling: ResponseFileHandling.Disabled);
+                enableTokenReplacement: false);
+            
             var parser = new Parser(configuration);
 
             var result = parser.Parse("@file.rsp");
@@ -322,9 +322,9 @@ public void When_response_file_processing_is_disabled_then_it_returns_response_f
         [Fact]
         public void Response_files_can_refer_to_other_response_files()
         {
-            var file3 = ResponseFile("--three", "3");
-            var file2 = ResponseFile($"@{file3}", "--two", "2");
-            var file1 = ResponseFile("--one", "1", $"@{file2}");
+            var file3 = CreateResponseFile("--three", "3");
+            var file2 = CreateResponseFile($"@{file3}", "--two", "2");
+            var file1 = CreateResponseFile("--one", "1", $"@{file2}");
 
             var option1 = new Option<int>("--one");
             var option2 = new Option<int>("--two");
@@ -339,17 +339,17 @@ public void Response_files_can_refer_to_other_response_files()
 
             var result = command.Parse($"@{file1}");
 
-            result.FindResultFor(option1).GetValueOrDefault().Should().Be(1);
-            result.FindResultFor(option1).GetValueOrDefault().Should().Be(1);
-            result.FindResultFor(option2).GetValueOrDefault().Should().Be(2);
-            result.FindResultFor(option3).GetValueOrDefault().Should().Be(3);
+            result.GetValueForOption(option1).Should().Be(1);
+            result.GetValueForOption(option1).Should().Be(1);
+            result.GetValueForOption(option2).Should().Be(2);
+            result.GetValueForOption(option3).Should().Be(3);
             result.Errors.Should().BeEmpty();
         }
 
         [Fact]
         public void When_response_file_options_or_arguments_contain_trailing_spaces_they_are_ignored()
         {
-            var responseFile = ResponseFile("--option1 ", "value1 ", "--option2\t", "2\t");
+            var responseFile = CreateResponseFile("--option1 ", "value1 ", "--option2\t", "2\t");
 
             var option1 = new Option<string>("--option1");
             var option2 = new Option<int>("--option2");
@@ -362,7 +362,7 @@ public void When_response_file_options_or_arguments_contain_trailing_spaces_they
         [Fact]
         public void When_response_file_options_or_arguments_contain_leading_spaces_they_are_ignored()
         {
-            var responseFile = ResponseFile(" --option1", " value1", "\t--option2", "\t2");
+            var responseFile = CreateResponseFile(" --option1", " value1", "\t--option2", "\t2");
 
             var option1 = new Option<string>("--option1");
             var option2 = new Option<int>("--option2");
@@ -376,7 +376,7 @@ public void When_response_file_options_or_arguments_contain_leading_spaces_they_
         [Fact]
         public void When_response_file_options_or_arguments_contain_trailing_and_leading_spaces_they_are_ignored()
         {
-            var responseFile = ResponseFile(" --option1 ", " value1 ", "\t--option2\t", "\t2\t");
+            var responseFile = CreateResponseFile(" --option1 ", " value1 ", "\t--option2\t", "\t2\t");
 
             var option1 = new Option<string>("--option1");
             var option2 = new Option<int>("--option2");