Combine merge tools (#977)

* Update clauses to separate unsafe code

For the grammar, we separate the unsafe code extensions. Make that distinction in the clauses.json, the code that reads it, and the section renumbering tool.

* Replicate current functionality

The section renumbering tool now replicates the output from the grammar extraction tool.

Refactoring to come.

* remove update grammar tool

It's no longer needed because it's functionality has been replicated in the section renumber tool.

* Refactoring, part 1

Simplify the loop that reads all the grammar productions from parts of the standard, and writes them to the grammar annex.

* refactoring, part 2

Read the existing text outside of the ANTLR productions from the existing annex.

That way, we can make any edits  to this text (including any changing section numbers) in the markdown file, rather than in the program source code.

* some final refactoring

* respond to feedback

* one final small lint fix

Author: BillWagner

.github/workflows/update-on-merge.yaml

@@ -53,13 +53,6 @@ jobs:
         ./run-section-renumber.sh
       shell: bash
 
-    - name: Update grammar annex
-      id: update-grammar
-      run: |
-        cd tools
-        ./update-grammar-annex.sh
-      shell: bash
-    
     - name: Create pull request
       uses: peter-evans/[email protected]
       if: ${{ steps.renumber-sections.outputs.status }} == 'success' && ${{ steps.update-grammar.outputs.status }} == 'success'

standard/clauses.json

@@ -3,13 +3,17 @@
     "foreword.md",
     "introduction.md"
   ],
-  "MainBody": [
+  "ScopeAndConformance": [
     "scope.md",
     "normative-references.md",
     "terms-and-definitions.md",
     "general-description.md",
-    "conformance.md",
-    "lexical-structure.md",
+    "conformance.md"
+  ],
+  "LexicalStructure": [
+    "lexical-structure.md"
+  ],
+  "MainBody": [
     "basic-concepts.md",
     "types.md",
     "variables.md",
@@ -25,7 +29,9 @@
     "enums.md",
     "delegates.md",
     "exceptions.md",
-    "attributes.md",
+    "attributes.md"
+  ],
+  "UnsafeClauses": [
     "unsafe-code.md"
   ],
   "Annexes": [

tools/GetGrammar/GetGrammar.csproj

@@ -0,0 +1,113 @@
+using System;
+using System.IO;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace StandardAnchorTags;
+
+public record GrammarHeaders(string LexicalHeader, 
+    string SyntacticHeader, 
+    string UnsafeExtensionsHeader,
+    string GrammarFooter);
+
+public class GenerateGrammar : IDisposable
+{
+    public static async Task<GrammarHeaders> ReadExistingHeaders(string pathToStandard, string grammarFile)
+    {
+        GrammarHeaders headers = new GrammarHeaders("", "", "", "");
+        StringBuilder headerBuffer = new StringBuilder();
+
+        using var reader = new StreamReader(Path.Combine(pathToStandard, grammarFile));
+
+        while (await reader.ReadLineAsync() is string inputLine)
+        {
+            headerBuffer.AppendLine(inputLine);
+            if (inputLine.StartsWith("```ANTLR"))
+            {
+                if (string.IsNullOrWhiteSpace(headers.LexicalHeader))
+                {
+                    headers = headers with { LexicalHeader = headerBuffer.ToString() };
+                }
+                else if (string.IsNullOrWhiteSpace(headers.SyntacticHeader))
+                {
+                    headers = headers with { SyntacticHeader = headerBuffer.ToString() };
+                }
+                else if (string.IsNullOrWhiteSpace(headers.UnsafeExtensionsHeader))
+                {
+                    headers = headers with { UnsafeExtensionsHeader = headerBuffer.ToString() };
+                }
+            } else if (inputLine.StartsWith("```"))
+            {
+                headerBuffer.Clear();
+                // Put the closing tag back:
+                headerBuffer.AppendLine(inputLine);
+            }
+        }
+        headers = headers with { GrammarFooter = headerBuffer.ToString() };
+        reader.Close();
+
+        return headers;
+    }
+
+    private readonly GrammarHeaders informativeTextBlocks;
+    private readonly string pathToStandardFiles;
+    private readonly StreamWriter grammarStream;
+
+    public GenerateGrammar(string grammarPath, string pathToStandardFiles, GrammarHeaders headers)
+    {
+        grammarStream = new StreamWriter(Path.Combine(pathToStandardFiles, grammarPath), false);
+        this.pathToStandardFiles = pathToStandardFiles;
+        informativeTextBlocks = headers;
+    }
+
+    public async Task WriteHeader() => await grammarStream.WriteAsync(informativeTextBlocks.LexicalHeader);
+    public async Task WriteSyntaxHeader() => await grammarStream.WriteAsync(informativeTextBlocks.SyntacticHeader);
+    public async Task WriteUnsafeExtensionHeader() => await grammarStream.WriteAsync(informativeTextBlocks.UnsafeExtensionsHeader);
+
+    public async Task WriteGrammarFooter()
+    {
+        await grammarStream.WriteAsync(informativeTextBlocks.GrammarFooter);
+        await grammarStream.FlushAsync();
+        grammarStream.Close();
+    }
+
+    public async Task ExtractGrammarFrom(string inputFileName)
+    {
+        string inputFilePath = $"{pathToStandardFiles}/{inputFileName}";
+        using var inputFile = new StreamReader(inputFilePath);
+        string section = "";
+        bool inProduction = false;
+
+        Console.OutputEncoding = Encoding.UTF8;
+
+        while (await inputFile.ReadLineAsync() is string inputLine)
+        {
+            if (inProduction)
+            {
+                if (inputLine.StartsWith("```"))
+                {
+                    inProduction = false;
+                }
+                else
+                {
+                    await grammarStream.WriteLineAsync(inputLine);
+                }
+            }
+            else
+            {
+                if (inputLine.StartsWith("#"))
+                {
+                    section = inputLine.Trim('#', ' ');
+                }
+                else if (inputLine.StartsWith("```ANTLR", StringComparison.InvariantCultureIgnoreCase))
+                {
+                    await grammarStream.WriteLineAsync();    // write out blank line before each new production
+                    await grammarStream.WriteLineAsync($"// Source: §{section}");
+                    inProduction = true;
+                }
+            }
+        }
+    }
+
+    public void Dispose() => grammarStream.Dispose();
+}