Add ignoreOutput option for example metadata (#724)

This allows the output to be ignored for nondeterministic or implementation-defined behavior.

Author: jskeet

standard/basic-concepts.md

@@ -968,7 +968,7 @@ The behavior of the garbage collector can be controlled, to some degree, via sta
 
 > *Example*: Since the garbage collector is allowed wide latitude in deciding when to collect objects and run finalizers, a conforming implementation might produce output that differs from that shown by the following code. The program
 >
-> <!-- ImplementationDefined$Example: {template:"standalone-console",name:"MemoryManagement1", expectedOutput:["xx"]} -->
+> <!-- Example: {template:"standalone-console",name:"MemoryManagement1", ignoreOutput:true} -->
 > <!-- Maintenance Note: The behavior of this example is implementation-defined. As such, the metadata does *not* compare test output with any ```console block. -->
 > ```csharp
 > class A
@@ -1023,7 +1023,7 @@ The behavior of the garbage collector can be controlled, to some degree, via sta
 >
 > In subtle cases, the distinction between “eligible for finalization” and “eligible for collection” can be important. For example,
 >
-> <!-- ImplementationDefined$Example: {template:"standalone-console",name:"MemoryManagement2", expectedOutput:["xx"]} -->
+> <!-- Example: {template:"standalone-console",name:"MemoryManagement2", ignoreOutput:true} -->
 > <!-- Maintenance Note: The behavior of this example is implementation-defined. As such, the metadata does *not* compare test output with any ```console block. -->
 > ```csharp
 > class A

tools/ExampleExtractor/Example.cs

@@ -105,11 +105,19 @@ private static IEnumerable<Example> LoadExamplesFromFile(string markdownFile)
             metadata.EndLine = closingLine;
             metadata.MarkdownFile = Path.GetFileName(markdownFile);
 
+            if (metadata.IgnoreOutput)
+            {
+                if (metadata.InferOutput || metadata.ExpectedOutput is not null)
+                {
+                    throw new InvalidOperationException($"Example {metadata.Name} has both {nameof(metadata.IgnoreOutput)} and either {nameof(metadata.InferOutput)} or {nameof(metadata.ExpectedOutput)}");
+                }
+            }
+
             if (metadata.InferOutput)
             {
                 if (metadata.ExpectedOutput is not null)
                 {
-                    throw new InvalidOperationException($"Example {metadata.Name} has both ${nameof(metadata.InferOutput)} and {nameof(metadata.ExpectedOutput)}");
+                    throw new InvalidOperationException($"Example {metadata.Name} has both {nameof(metadata.InferOutput)} and {nameof(metadata.ExpectedOutput)}");
                 }
                 int openingConsoleLine = FindLineEnding(closingLine + 1, "```console");
                 // We expect the output to appear very shortly after the example.

tools/ExampleExtractor/ExampleMetadata.cs

@@ -46,6 +46,14 @@ public class ExampleMetadata
     [JsonProperty("inferOutput")]
     public bool InferOutput { get; set; }
 
+    /// <summary>
+    /// If this is set, ExpectedOutput must be null and InferOutput must be false.
+    /// The actual output is then ignored by the test runner.
+    /// This option should be used when output is nondeterministic.
+    /// </summary>
+    [JsonProperty("ignoreOutput")]
+    public bool IgnoreOutput { get; set; }
+
     [JsonProperty("expectedException")]
     public string ExpectedException { get; set; }
 

tools/ExampleTester/GeneratedExample.cs

@@ -161,7 +161,7 @@ bool ValidateOutput()
             }
             var expectedLines = Metadata.ExpectedOutput ?? new List<string>();
             return ValidateException(actualException, Metadata.ExpectedException) &&
-                ValidateExpectedAgainstActual("output", expectedLines, actualLines);
+                (Metadata.IgnoreOutput || ValidateExpectedAgainstActual("output", expectedLines, actualLines));
         }
 
         bool ValidateException(Exception? actualException, string? expectedExceptionName)